diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..e9b7d562 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,15 @@ +# Set the default behavior, in case people don't have `core.autocrlf` set. +* text=auto + +# Explicitly declare text files you want to always be normalized and converted +# to native OS line endings on checkout, but committing those with LF in the repo. + +# Required for robot tests to run properly in Windows. +**/testdata/*.txt text eol=lf +**/testdata/*.yaml text eol=lf + +*.py text +*.js text +*.md text +*.robot text +*.go text diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 00000000..b0330eac --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,72 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL" + +on: + push: + branches: [ "master" ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ "master" ] + schedule: + - cron: '24 10 * * 1' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ 'go' ] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] + # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + + # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs + # queries: security-extended,security-and-quality + + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v3 + + # ℹī¸ Command-line programs to run using the OS shell. + # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun + + # If the Autobuild fails above, remove it and uncomment the following three lines. + # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. + + # - run: | + # echo "Run, Build Application using script" + # ./location_of_script_within_repo/buildscript.sh + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 diff --git a/.github/workflows/generator-generic-ossf-slsa3-publish.yml b/.github/workflows/generator-generic-ossf-slsa3-publish.yml new file mode 100644 index 00000000..35c829b1 --- /dev/null +++ b/.github/workflows/generator-generic-ossf-slsa3-publish.yml @@ -0,0 +1,66 @@ +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +# This workflow lets you generate SLSA provenance file for your project. +# The generation satisfies level 3 for the provenance requirements - see https://slsa.dev/spec/v0.1/requirements +# The project is an initiative of the OpenSSF (openssf.org) and is developed at +# https://github.com/slsa-framework/slsa-github-generator. +# The provenance file can be verified using https://github.com/slsa-framework/slsa-verifier. +# For more information about SLSA and how it improves the supply-chain, visit slsa.dev. + +name: SLSA generic generator +on: + workflow_dispatch: + release: + types: [created] + +jobs: + build: + runs-on: ubuntu-latest + outputs: + digests: ${{ steps.hash.outputs.digests }} + + steps: + - uses: actions/checkout@v4 + + # ======================================================== + # + # Step 1: Build your artifacts. + # + # ======================================================== + - name: Build artifacts + run: | + # These are some amazing artifacts. + echo "artifact1" > artifact1 + echo "artifact2" > artifact2 + + # ======================================================== + # + # Step 2: Add a step to generate the provenance subjects + # as shown below. Update the sha256 sum arguments + # to include all binaries that you generate + # provenance for. + # + # ======================================================== + - name: Generate subject for provenance + id: hash + run: | + set -euo pipefail + + # List the artifacts the provenance will refer to. + files=$(ls artifact*) + # Generate the subjects (base64 encoded). + echo "hashes=$(sha256sum $files | base64 -w0)" >> "${GITHUB_OUTPUT}" + + provenance: + needs: [build] + permissions: + actions: read # To read the workflow path. + id-token: write # To sign the provenance. + contents: write # To add assets to a release. + uses: slsa-framework/slsa-github-generator/.github/workflows/generator_generic_slsa3.yml@v1.4.0 + with: + base64-subjects: "${{ needs.build.outputs.digests }}" + upload-assets: true # Optional: Upload to a new release diff --git a/.github/workflows/rcc.yaml b/.github/workflows/rcc.yaml index 34cb6abd..01ec0a33 100644 --- a/.github/workflows/rcc.yaml +++ b/.github/workflows/rcc.yaml @@ -1,47 +1,76 @@ name: Rcc on: - push: - branches: - - master - - maintenance + workflow_dispatch: + # enables manual triggering + push: + branches: + - master + - maintenance + - series10 + pull_request: + branches: + - master jobs: - build: - name: Build - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v1 - - uses: actions/setup-ruby@v1 - with: - ruby-version: '2.5' - - name: What - run: rake what - - name: Building - run: rake build + build: + name: Build + runs-on: ubuntu-latest + steps: + - uses: actions/setup-go@v5 + with: + go-version: "1.20.x" + - name: Set up Python 3.10 + uses: actions/setup-python@v5 + with: + python-version: "3.10" + - uses: actions/checkout@v4 + - name: Install invoke + run: python -m pip install invoke + - name: What + run: inv what + - name: Building + run: inv build - robot: - name: Robot - runs-on: ${{ matrix.os }}-latest - strategy: - fail-fast: false - matrix: - os: ['ubuntu'] - steps: - - uses: actions/checkout@v1 - - uses: actions/setup-ruby@v1 - with: - ruby-version: '2.5' - - uses: actions/setup-python@v1 - with: - python-version: '3.7' - - name: Setup - run: rake robotsetup - - name: What - run: rake what - - name: Testing - run: rake robot - - uses: actions/upload-artifact@v1 - if: success() || failure() - with: - name: ${{ matrix.os }}-test-reports - path: ./tmp/output/ + robot: + name: Robot + runs-on: ${{ matrix.os }}-latest + strategy: + fail-fast: false + matrix: + os: ["ubuntu", "windows"] + steps: + - uses: actions/setup-go@v5 + with: + go-version: "1.20.x" + - uses: actions/setup-python@v5 + with: + python-version: "3.10" + - uses: actions/checkout@v4 + - name: Install invoke + run: python -m pip install invoke + - name: Setup + run: inv robotsetup + - name: What + run: inv what + - name: Testing + run: inv robot + - uses: actions/upload-artifact@v4 + if: success() || failure() + with: + name: ${{ matrix.os }}-test-reports + path: ./tmp/output/ + + trigger: + name: Trigger + runs-on: ubuntu-latest + needs: + - build + - robot + if: success() && github.ref == 'refs/heads/master' + steps: + - name: Pipeline + run: | + curl -X POST https://api.github.com/repos/robocorp/rcc-pipeline/dispatches \ + -H 'Accept: application/vnd.github.v3+json' \ + -u ${{ secrets.TRIGGER_TOKEN }} \ + --data '{"event_type": "pipes"}' diff --git a/.gitignore b/.gitignore index 6df2bba6..93c93360 100644 --- a/.gitignore +++ b/.gitignore @@ -20,7 +20,10 @@ __pycache__/ build/ output/ tmp/ +developer/tmp/ rcc.yaml rcccache.yaml tags .DS_Store +.use +.idea/ diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 00000000..edf0efca --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,36 @@ +{ + "python.pydev.pythonPath": [ + ".", + ], + "python.pydev.preferredImportLocation": "topOfMethod", + + "python.pydev.formatter": "ruff", + "python.pydev.lint.ruff.use": true, + "python.pydev.lint.ruff.showOutput": true, + "python.pydev.lint.mypy.use": true, + "python.pydev.lint.mypy.showOutput": true, + "python.pydev.lint.mypy.args": "--follow-imports=silent --show-column-numbers --namespace-packages --explicit-package-bases", + "python.pydev.docstring.style": "google", + "editor.formatOnType": true, + "editor.formatOnSave": true, + "python.pydev.sortImportsOnFormat": "isort", + + "[typescript]": { + "editor.defaultFormatter": "esbenp.prettier-vscode" + }, + "[markdown]": { + "editor.defaultFormatter": "esbenp.prettier-vscode" + }, + "[javascript]": { + "editor.defaultFormatter": "esbenp.prettier-vscode" + }, + "[jsonc]": { + "editor.defaultFormatter": "esbenp.prettier-vscode" + }, + "[html]": { + "editor.defaultFormatter": "vscode.html-language-features" + }, + "[typescriptreact]": { + "editor.defaultFormatter": "esbenp.prettier-vscode" + } +} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..d5e425d8 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,45 @@ +# How to contribute + +## Ideas for regular sources of contribution. + +1. Is there a need to upgrade the Go language version? + - see releases from https://go.dev/doc/devel/release + - do not go to bleeding edge (unless you really have to do so) + - do not stay too far away behind development + - when needed, update .github/workflows/rcc.yaml +2. Is there new Micromamba available? + - see releases from https://anaconda.org/conda-forge/micromamba + - only use stable versions +3. Where is uv going? + - important: uv has to come from conda-forge because of enterprise + firewalls/proxies + - https://anaconda.org/conda-forge/uv to find out what is available + on conda-forge + - https://github.com/astral-sh/uv to see issues and development +4. What is pip doing? + - check news from https://pip.pypa.io/en/stable/news/ + +## Additional sources for contribution ideas. + +- improve documentation under docs/ directory +- improve acceptance tests written in Robot Framework (inside `robot_tests` + directory) + - currently these work fully on Linux only, so if you have Mac or Windows + and can make these work there, that would also be a nice contribution + +## How to proceed with improvements/contributions? + +- create an issue in the rcc repository at + https://github.com/robocorp/rcc/issues +- on that issue, discuss the solution you are proposing +- implementation can proceed only when the solution is clear and accepted +- the solution should be made so that it works on Mac, Windows, and Linux +- when developing, remember to run both unit tests and acceptance tests + (Robot Framework tests) on your own machine first +- once you have written the code for that solution, create a pull request + +## How does rcc build work? + +- a good source to understand the build is to see the CI pipeline, + .github/workflows/rcc.yaml +- also read docs/BUILD.md for tooling requirements and commands to run diff --git a/README.md b/README.md index e187301c..a284177f 100644 --- a/README.md +++ b/README.md @@ -1,50 +1,92 @@ ![RCC](/docs/title.png) -RCC is a set of tooling that allows you to create, manage, and distribute Python-based self-contained automation packages - or robots :robot: as we call them. +RCC allows you to create, manage, and distribute Python-based self-contained automation packages. RCC also allows you to run your automations in isolated Python environments so they can still access the rest of your machine. -Together with [robot.yaml](https://robocorp.com/docs/setup/robot-yaml-format) configuration file, `rcc` is a foundation that allows anyone to build and share automation with ease. +🚀 "Repeatable, movable and isolated Python environments for your automation." + +Together with [robot.yaml](https://robocorp.com/docs/robot-structure/robot-yaml-format) configuration file, `rcc` is a foundation that allows anyone to build and share automation easily.

+RCC is actively maintained by [Sema4.ai](https://sema4.ai/). + + +## Why use rcc? + +* You do not need to install Python on the target machine +* You can control exactly which version of Python your automation will run on (..and which pip version is used to resolve dependencies) +* You can avoid `Works on my machine` +* No need for `venv`, `pyenv`, ... tooling and knowledge sharing inside your team. +* Define dependencies in `conda.yaml` and automation config in `robot.yaml` and let RCC do the heavy lifting. +* If you have run into "dependency drifts", where once working runtime environment dependencies get updated and break your production system?, RCC can freeze ALL dependencies, pre-build environments, and more. +* RCC will give you a heads-up if your automations have been leaving behind processes after running. + +...and much much more. + +👉 If the command line seems scary, just pick up [Robocorp Code](https://marketplace.visualstudio.com/items?itemName=robocorp.robocorp-code) -extension for VS Code, and you'll get the power of RCC directly in VS Code without worrying about the commands. ## Getting Started :arrow_double_down: Install rcc -> [Download RCC](#direct-downloads-for-signed-executables-provided-by-robocorp) +> [Installation guide](https://github.com/robocorp/rcc?tab=readme-ov-file#installing-rcc-from-the-command-line) :octocat: Pull robot from GitHub: -> `rcc pull github.com/robocorp/example-google-image-search` +> `rcc pull github.com/robocorp/template-python-browser` :running: Run robot > `rcc run` -:hatching_chick: Create your own robot from template -> `rcc robot initialize -t standard` +:hatching_chick: Create your own robot from templates +> `rcc create` + +For detailed instructions, visit [Robocorp RCC documentation](https://robocorp.com/docs/rcc/overview) to get started. To build `rcc` from this repository, see the [Setup Guide](/docs/BUILD.md) + +## Installing RCC from the command line + +> Links to changelog and different versions [available here](https://downloads.robocorp.com/rcc/releases/index.html) -For detailed instructions, visit [Robocorp RCC documentation](https://robocorp.com/docs/product-manuals/robocorp-cli) to get started. To build `rcc` from this repository see the [Setup Guide](/docs/BUILD.md) +### Windows -### Direct downloads for signed executables provided by Robocorp +1. Open the command prompt +1. Download: `curl -o rcc.exe https://downloads.robocorp.com/rcc/releases/latest/windows64/rcc.exe` +1. [Add to system path](https://www.architectryan.com/2018/03/17/add-to-the-path-on-windows-10/): Open Start -> `Edit the system environment variables` +1. Test: `rcc` -| OS | Download URL | -| ------- | ---------------------------------------------------------------- | -| Windows | https://downloads.code.robocorp.com/rcc/latest/windows64/rcc.exe | -| macOS | https://downloads.code.robocorp.com/rcc/latest/macos64/rcc | -| Linux | https://downloads.code.robocorp.com/rcc/latest/linux64/rcc | +### macOS -*[EULA for pre-built distribution.](https://cdn.robocorp.com/legal/Robocorp-EULA-v1.0.pdf)* +#### Brew cask from Robocorp tap + +1. Update brew: `brew update` +1. Install: `brew install robocorp/tools/rcc` +1. Test: `rcc` + +Upgrading: `brew upgrade rcc` + +### Linux + +1. Open the terminal +1. Download: `curl -o rcc https://downloads.robocorp.com/rcc/releases/latest/linux64/rcc` +1. Make the downloaded file executable: `chmod a+x rcc` +1. Add to path: `sudo mv rcc /usr/local/bin/` +1. Test: `rcc` ## Documentation Visit [https://robocorp.com/docs](https://robocorp.com/docs) to view the full documentation on the full Robocorp stack. -## Community +The changelog can be seen [here](/docs/changelog.md). It is also visible inside RCC using the command `rcc docs changelog`. -The Robocorp community can be found on [Developer Slack](https://robocorp-developers.slack.com), where you can ask questions, voice ideas, and share your projects. +[EULA for pre-built distribution.](https://cdn.robocorp.com/legal/Robocorp-EULA-v1.0.pdf) + +Some tips, tricks, and recipes can be found [here](/docs/recipes.md). +These are also visible inside RCC using the command: `rcc docs recipes`. -You can also use the [Robocorp Forum](https://forum.robocorp.com) +## Community and Support + +The Robocorp community can be found on [Developer Slack](https://robocorp-developers.slack.com), where you can ask questions, voice ideas, and share your projects. ## License -Apache 2.0 \ No newline at end of file +Apache 2.0 diff --git a/Rakefile b/Rakefile deleted file mode 100644 index 4596c6f8..00000000 --- a/Rakefile +++ /dev/null @@ -1,111 +0,0 @@ -if Rake::Win32.windows? then - PYTHON='python' - LS='dir' -else - PYTHON='python3' - LS='ls -l' -end - -desc 'Show latest HEAD with stats' -task :what do - sh 'go version' - sh 'git --no-pager log -2 --stat HEAD' -end - -task :tooling do - puts "PATH is #{ENV['PATH']}" - puts "GOPATH is #{ENV['GOPATH']}" - puts "GOROOT is #{ENV['GOROOT']}" - sh "go get -u github.com/go-bindata/go-bindata/..." - sh "which -a zip || echo NA" - sh "which -a go-bindata || echo NA" - sh "ls -l $HOME/go/bin" -end - -task :assets do - FileList['templates/*/'].each do |directory| - basename = File.basename(directory) - assetname = File.absolute_path(File.join("assets", "#{basename}.zip")) - rm_rf assetname - puts "Directory #{directory} => #{assetname}" - sh "cd #{directory} && zip -ryqD9 #{assetname} ." - end - sh "$HOME/go/bin/go-bindata -o blobs/assets.go -pkg blobs assets/*.zip assets/man/*" -end - -task :support do - sh 'mkdir -p tmp build/linux64 build/linux32 build/macos64 build/windows64 build/windows32' -end - -desc 'Run tests.' -task :test => [:support, :assets] do - sh 'go test -cover -coverprofile=tmp/cover.out ./...' - sh 'go tool cover -func=tmp/cover.out' -end - -task :linux64 => [:what, :test] do - ENV['GOOS'] = 'linux' - ENV['GOARCH'] = 'amd64' - sh "go build -ldflags '-s' -o build/linux64/ ./cmd/..." - sh "sha256sum build/linux64/* || true" -end - -task :linux32 => [:what, :test] do - ENV['GOOS'] = 'linux' - ENV['GOARCH'] = '386' - sh "go build -ldflags '-s' -o build/linux32/ ./cmd/..." - sh "sha256sum build/linux32/* || true" -end - -task :macos64 => [:support] do - ENV['GOOS'] = 'darwin' - ENV['GOARCH'] = 'amd64' - sh "go build -ldflags '-s' -o build/macos64/ ./cmd/..." - sh "sha256sum build/macos64/* || true" -end - -task :windows64 => [:support] do - ENV['GOOS'] = 'windows' - ENV['GOARCH'] = 'amd64' - sh "go build -ldflags '-s' -o build/windows64/ ./cmd/..." - sh "sha256sum build/windows64/* || true" -end - -task :windows32 => [:support] do - ENV['GOOS'] = 'windows' - ENV['GOARCH'] = '386' - sh "go build -ldflags '-s' -o build/windows32/ ./cmd/..." - sh "sha256sum build/windows32/* || true" -end - -desc 'Setup build environment' -task :robotsetup do - sh "#{PYTHON} -m pip install --upgrade -r robot_requirements.txt" - sh "#{PYTHON} -m pip freeze" -end - -desc 'Build local, operating system specific rcc' -task :local => [:tooling, :support, :assets] do - sh "go build -o build/ ./cmd/..." -end - -desc 'Run robot tests on local application' -task :robot => :local do - sh "robot -L DEBUG -d tmp/output robot_tests" -end - -desc 'Build commands to linux, macos, and windows.' -task :build => [:tooling, :version_txt, :linux64, :linux32, :macos64, :windows64, :windows32] do - sh 'ls -l $(find build -type f)' -end - -def version - `sed -n -e '/Version/{s/^.*\`v//;s/\`$//p}' common/version.go`.strip -end - -task :version_txt => :support do - File.write('build/version.txt', "v#{version}") -end - -task :default => :build - diff --git a/anywork/pending.go b/anywork/pending.go new file mode 100644 index 00000000..a677fdd5 --- /dev/null +++ b/anywork/pending.go @@ -0,0 +1,74 @@ +package anywork + +type ( + WorkGroup interface { + add() + done() + Wait() + } + + workgroup struct { + level waitpipe + waiting waiting + } + + waitpipe chan bool + waiting chan waitpipe +) + +func NewGroup() WorkGroup { + group := &workgroup{ + level: make(waitpipe), + waiting: make(waiting), + } + go group.waiter() + return group +} + +func (it *workgroup) add() { + it.level <- true +} + +func (it *workgroup) done() { + it.level <- false +} + +func (it *workgroup) Wait() { + reply := make(waitpipe) + it.waiting <- reply + _, _ = <-reply +} + +func (it *workgroup) waiter() { + var counter int64 + pending := make([]waitpipe, 0, 5) +forever: + for { + if counter < 0 { + panic("anywork: counter below zero") + } + if counter == 0 && len(pending) > 0 { + for _, waiter := range pending { + close(waiter) + } + pending = make([]waitpipe, 0, 5) + } + select { + case up, ok := <-it.level: + if !ok { + break forever + } + if up { + counter += 1 + } else { + counter -= 1 + } + case waiter, ok := <-it.waiting: + if !ok { + break forever + } + pending = append(pending, waiter) + } + } + panic("anywork: for some reason, waiter have just exited") +} diff --git a/anywork/worker.go b/anywork/worker.go new file mode 100644 index 00000000..47831fb5 --- /dev/null +++ b/anywork/worker.go @@ -0,0 +1,121 @@ +package anywork + +import ( + "fmt" + "io" + "os" + "runtime" +) + +var ( + group WorkGroup + pipeline WorkQueue + failpipe Failures + errcount Counters + headcount uint64 + WorkerCount int +) + +type Work func() +type WorkQueue chan Work +type Failures chan string +type Counters chan uint64 + +func catcher(title string, identity uint64) { + catch := recover() + if catch != nil { + failpipe <- fmt.Sprintf("Recovering %q #%d: %v", title, identity, catch) + } +} + +func process(fun Work, identity uint64) { + defer catcher("process", identity) + fun() +} + +func member(identity uint64) { + defer catcher("member", identity) + for { + work, ok := <-pipeline + if !ok { + break + } + process(work, identity) + group.done() + } +} + +func watcher(failures Failures, counters Counters) { + counter := uint64(0) + for { + select { + case fail := <-failures: + counter += 1 + fmt.Fprintln(os.Stderr, fail) + case counters <- counter: + counter = 0 + } + } +} + +func init() { + group = NewGroup() + pipeline = make(WorkQueue, 100000) + failpipe = make(Failures) + errcount = make(Counters) + headcount = 0 + AutoScale() + go watcher(failpipe, errcount) +} + +func Scale() uint64 { + return headcount +} + +func AutoScale() { + limit := uint64(runtime.NumCPU() - 1) + if WorkerCount > 1 { + limit = uint64(WorkerCount) + } + if limit > 96 { + limit = 96 + } + if limit < 2 { + limit = 2 + } + for headcount < limit { + go member(headcount) + headcount += 1 + } +} + +func Backlog(todo Work) { + if todo != nil { + group.add() + pipeline <- todo + } +} + +func Sync() error { + trials := int(Scale()) + for retries := 0; retries < trials; retries++ { + runtime.Gosched() + } + group.Wait() + count := <-errcount + if count > 0 { + return fmt.Errorf("There has been %d failures. See messages above.", count) + } + return nil +} + +func OnErrPanicCloseAll(err error, closers ...io.Closer) { + if err != nil { + for _, closer := range closers { + if closer != nil { + closer.Close() + } + } + panic(err) + } +} diff --git a/anywork/worker_test.go b/anywork/worker_test.go new file mode 100644 index 00000000..b748d05a --- /dev/null +++ b/anywork/worker_test.go @@ -0,0 +1 @@ +package anywork_test diff --git a/assets/depxtraction.py b/assets/depxtraction.py new file mode 100755 index 00000000..6c644737 --- /dev/null +++ b/assets/depxtraction.py @@ -0,0 +1,87 @@ +#!/bin/env python3 + +import pip, re, sys + +from collections import namedtuple +from importlib import metadata + +REJECTED = {'pip', 'pkg_resources', 'pkgutil_resolve_name', 'setuptools', 'wheel'} +CONDA_FORGE = {'robocorp-truststore'} +EMPTY = tuple() + +NAMEFORM = re.compile(r'^([a-z0-9](?:[a-z0-9._-]*?[a-z0-9])?)([^a-z0-9._-].*)?$', re.I) +EXTRAFORM = re.compile(r'\bextra\s*=') +NORMALIZE = re.compile(r'[-_.]+') + +Metadata = namedtuple('Metadata', 'key name version needs') + +HEADER = f''' +# This was generated by `{__file__}` script. +# This is experimental feature for getting `environment.yaml` from installed base. +# It only supports simple `pip` environments, where most dependencies are coming +# from pypi.org. If your desire is to have addtional packages from `conda-forge` +# those must be maintained manually on above `pip:` section. + +channels: +- conda-forge +dependencies: +'''.strip() + +FOOTER = ''' +# NOTE: building above environment might fail, because dependencies are +# collected heuristically. So once you have generated new environment +# configuration, you should actually try to build it using `rcc`. +''' + +def normalize(name): + return NORMALIZE.sub('-', str(name)).lower().strip() + return re.sub(r'[-_.]+', '-', str(name)).lower().strip() + +def list_modules(): + for candidate in metadata.distributions(): + yield candidate + +def parselet(text): + head, *rest = map(str.strip, text.split(';')) + name, *ignore = map(str.strip, filter(bool, NAMEFORM.match(head).groups())) + extra = any(map(EXTRAFORM.match, rest)) + return extra, normalize(name) + +def environment_yaml(resolved): + print(HEADER) + python = sys.version_info + print(f'- python={python.major}.{python.minor}.{python.micro}') + print(f'- pip={pip.__version__}') + for conda in CONDA_FORGE: + if version := resolved.pop(conda, None): + print(f'- {conda}={version}') + if resolved: + print(f'- pip:') + for name, version in resolved.items(): + print(f' - {name}=={version}') + print(FOOTER) + +def process(): + metadata = dict() + for module in list_modules(): + name = module.metadata.get('name') + key = normalize(name) + metadata[key] = Metadata(key, name, module.version, module.requires or EMPTY) + + cyclic = set() + toplevel = set(metadata.keys()) + tuple(map(toplevel.discard, REJECTED)) + for package, needs in sorted(metadata.items()): + for entry in needs.needs: + rejected, name = parselet(entry) + if (package, name) in cyclic: + continue + if not rejected: + cyclic.add((name, package)) + toplevel.discard(name) + + resolved = dict([metadata[x].name, metadata[x].version] for x in sorted(toplevel)) + environment_yaml(resolved) + +if __name__ == '__main__': + process() diff --git a/assets/externally_managed.txt b/assets/externally_managed.txt new file mode 100644 index 00000000..62350831 --- /dev/null +++ b/assets/externally_managed.txt @@ -0,0 +1,21 @@ +[externally-managed] +Error=by Robocorp tooling called `rcc`. + + To install Python packages into this managed environment, those should + be added to `conda.yaml` file, or more generally into "environment + configuration files". + + Motivation with these kind of managed environments is, that they provide: + - repeatability, so that same environment can be recreated later + - isolation, so that different automations can have their own dependencies + - as few as possible dependency resolutions (currently two: conda and pypi) + - support for "foreign machines", and not just your own personal machine + - supporting Windows, MacOS, and linux operating systems with same automations + + If you don't need above features, or need more flexibility in your personal + developement environment, consider using something else (like virtualenv or + poetry) with your personal tooling, and only use `rcc` managed environments + for final delivery to your users and customers. + + For more details, see: + https://github.com/robocorp/rcc/blob/master/docs/recipes.md#what-are-environmentconfigs diff --git a/assets/man/tutorial.txt b/assets/man/tutorial.txt index 8349bd78..bff88f6b 100644 --- a/assets/man/tutorial.txt +++ b/assets/man/tutorial.txt @@ -1,22 +1,25 @@ -Road to Robocorp Command Center (RCC) -===================================== +# Welcome to RCC tutorial -First, download a bunch of example robots using following command. +Create you first Robot Framework or python robot and follow given instructions to run it: - rcc pull example-activities + rcc create -Then change into some robot root directory. +Easy way to run existing, unwrapped robot is just like this: - cd example-activities-master/google-image-search + rcc run --robot path/to/my/robot.yaml --task "My fine task" -Now you are ready to run your first robot. +Jumpstart your development with robot examples and templates from our community. +Download and run any robot from Robocorp Portal https://robocorp.com/robots/ +For example: + rcc pull example-google-image-search + cd example-google-image-search-main rcc run -Explore other available commands. +Check help for any command with --help. Explore other available commands. rcc - rcc robot -h - rcc robot initialize -h + rcc configure credentials + rcc cloud push -Have fun! ;-) +Have fun! diff --git a/assets/micromamba_version.txt b/assets/micromamba_version.txt new file mode 100644 index 00000000..1097babb --- /dev/null +++ b/assets/micromamba_version.txt @@ -0,0 +1 @@ +v1.5.8 diff --git a/assets/netdiag.yaml b/assets/netdiag.yaml new file mode 100644 index 00000000..ad9aff6f --- /dev/null +++ b/assets/netdiag.yaml @@ -0,0 +1,44 @@ +network: + dns-lookup: + - www.robocorp.com + tls-verify: + - chat.robocorp.com + head-request: + - url: https://www.robocorp.com + codes: [200] + - url: https://downloads.robocorp.com/canary.txt + codes: [200] + - url: https://pypi.org/simple/jupyterlab-pygments/ + codes: [200] + - url: https://conda.anaconda.org/conda-forge/linux-64/repodata.json + codes: [200] + - url: https://cloud.robocorp.com + codes: [200] + - url: https://api.eu1.robocorp.com + codes: [403] + - url: https://api.eu1.robocloud.eu + codes: [403] + - url: https://roboworker-control-ws-v2.eu1.robocloud.eu + codes: [403] + - url: https://task-data-api.eu1.robocloud.eu + codes: [403] + - url: https://telemetry.robocorp.com + codes: [403] + - url: https://feedback.robocorp.com + codes: [403] + - url: https://status.robocorp.com + codes: [200] + - url: https://status.robocorp.com/history.atom + codes: [200] + - url: https://status.robocorp.com/history.rss + codes: [200] + - url: https://status.robocorp.com/uptime + codes: [200] + - url: https://robocorp.com/docs + codes: [200] + - url: https://robocorp.com/portal + codes: [200] + get-request: + - url: https://downloads.robocorp.com/canary.txt + codes: [200] + content-sha256: 7a8b721c71428e8599d250e647135550c05a71cc50276e9eea09d82d1baf09a1 diff --git a/assets/robocorp_settings.yaml b/assets/robocorp_settings.yaml new file mode 100644 index 00000000..44568d27 --- /dev/null +++ b/assets/robocorp_settings.yaml @@ -0,0 +1,46 @@ +endpoints: + cloud-api: https://api.eu1.robocorp.com/ + cloud-linking: https://cloud.robocorp.com/link/ + cloud-ui: https://cloud.robocorp.com/ + pypi: # https://pypi.org/simple/ + pypi-trusted: # https://pypi.org/ + conda: # https://conda.anaconda.org/ + downloads: https://downloads.robocorp.com/ + docs: https://robocorp.com/docs/ + telemetry: https://telemetry.robocorp.com/ + issues: https://telemetry.robocorp.com/ + +diagnostics-hosts: + - files.pythonhosted.org + - github.com + - conda.anaconda.org + - pypi.org + +autoupdates: + assistant: https://downloads.robocorp.com/assistant/releases/ + workforce-agent: https://downloads.robocorp.com/workforce-agent/releases/ + setup-utility: https://downloads.robocorp.com/setup-utility/releases/ + templates: https://downloads.robocorp.com/templates/templates.yaml + +certificates: + verify-ssl: true + ssl-no-revoke: false + legacy-renegotiation-allowed: false + +options: + no-build: false + +network: + no-proxy: # no no proxy by default + https-proxy: # no proxy by default + http-proxy: # no proxy by default + +branding: + logo: https://downloads.robocorp.com/company/press-kit/logos/robocorp-logo-black.svg + theme-color: FF0000 + +meta: + name: default + description: Robocorp.com default settings.yaml internal to rcc + source: builtin + version: 2023.09 diff --git a/assets/sema4ai_settings.yaml b/assets/sema4ai_settings.yaml new file mode 100644 index 00000000..448738a6 --- /dev/null +++ b/assets/sema4ai_settings.yaml @@ -0,0 +1,36 @@ +endpoints: + cloud-api: https://api.eu1.robocorp.com/ + cloud-linking: https://cloud.robocorp.com/link/ + cloud-ui: https://cloud.robocorp.com/ + downloads: https://cdn.sema4.ai/ + docs: https://sema4.ai/docs/ + telemetry: https://sema4.ai/api/telemetry + issues: https://telemetry.robocorp.com/ + +diagnostics-hosts: + - files.pythonhosted.org + - github.com + - conda.anaconda.org + - pypi.org + +autoupdates: + templates: https://cdn.sema4.ai/templates/templates.yaml + +certificates: + verify-ssl: true + ssl-no-revoke: false + legacy-renegotiation-allowed: false + +options: + no-build: false + +network: + no-proxy: # no no proxy by default + https-proxy: # no proxy by default + http-proxy: # no proxy by default + +meta: + name: default + description: Sema4.ai default settings.yaml internal to rcc + source: builtin + version: 2024.06 diff --git a/assets/speedtest.yaml b/assets/speedtest.yaml new file mode 100644 index 00000000..099faea0 --- /dev/null +++ b/assets/speedtest.yaml @@ -0,0 +1,7 @@ +channels: +- conda-forge +dependencies: +- python=3.9.13 +- pip=22.1.2 +- pip: + - robotframework==4.1.2 diff --git a/assets/templates.yaml b/assets/templates.yaml new file mode 100644 index 00000000..d22ca4ad --- /dev/null +++ b/assets/templates.yaml @@ -0,0 +1,4 @@ +templates: + standard: Standard Robot Framework template + extended: Extended Robot Framework template + python: Basic Python template diff --git a/blobs/.gitignore b/blobs/.gitignore deleted file mode 100644 index 60ce659f..00000000 --- a/blobs/.gitignore +++ /dev/null @@ -1 +0,0 @@ -assets.go diff --git a/blobs/asset_test.go b/blobs/asset_test.go index fad32bc3..2e42bf08 100644 --- a/blobs/asset_test.go +++ b/blobs/asset_test.go @@ -23,10 +23,39 @@ func TestCanSeeBaseZipAsset(t *testing.T) { wont_be.Nil(asset) } +func TestCanOtherAssets(t *testing.T) { + must_be, wont_be := hamlet.Specifications(t) + + must_be.Panic(func() { blobs.MustAsset("assets/missing.yaml") }) + must_be.Panic(func() { blobs.MustAsset("assets/settings.yaml") }) + + wont_be.Panic(func() { blobs.MustAsset("assets/robocorp_settings.yaml") }) + wont_be.Panic(func() { blobs.MustAsset("assets/sema4ai_settings.yaml") }) + + wont_be.Panic(func() { blobs.MustAsset("assets/micromamba_version.txt") }) + wont_be.Panic(func() { blobs.MustAsset("assets/externally_managed.txt") }) + + wont_be.Panic(func() { blobs.MustAsset("assets/templates.yaml") }) + wont_be.Panic(func() { blobs.MustAsset("assets/speedtest.yaml") }) + + wont_be.Panic(func() { blobs.MustAsset("assets/man/LICENSE.txt") }) + wont_be.Panic(func() { blobs.MustAsset("assets/man/tutorial.txt") }) + + wont_be.Panic(func() { blobs.MustAsset("docs/BUILD.md") }) + wont_be.Panic(func() { blobs.MustAsset("docs/README.md") }) + wont_be.Panic(func() { blobs.MustAsset("docs/changelog.md") }) + wont_be.Panic(func() { blobs.MustAsset("docs/environment-caching.md") }) + wont_be.Panic(func() { blobs.MustAsset("docs/features.md") }) + wont_be.Panic(func() { blobs.MustAsset("docs/profile_configuration.md") }) + wont_be.Panic(func() { blobs.MustAsset("docs/recipes.md") }) + wont_be.Panic(func() { blobs.MustAsset("docs/usecases.md") }) + wont_be.Panic(func() { blobs.MustMicromamba() }) +} + func TestCanGetTemplateNamesThruOperations(t *testing.T) { must_be, wont_be := hamlet.Specifications(t) - assets := operations.ListTemplates() + assets := operations.ListTemplates(true) wont_be.Nil(assets) must_be.True(len(assets) == 3) must_be.Equal([]string{"extended", "python", "standard"}, assets) diff --git a/blobs/assets/.gitignore b/blobs/assets/.gitignore new file mode 100644 index 00000000..3b54ee8b --- /dev/null +++ b/blobs/assets/.gitignore @@ -0,0 +1,5 @@ +*.zip +*.yaml +*.txt +*.py +micromamba* diff --git a/blobs/assets/man/.gitignore b/blobs/assets/man/.gitignore new file mode 100644 index 00000000..2211df63 --- /dev/null +++ b/blobs/assets/man/.gitignore @@ -0,0 +1 @@ +*.txt diff --git a/blobs/docs/.gitignore b/blobs/docs/.gitignore new file mode 100644 index 00000000..dd449725 --- /dev/null +++ b/blobs/docs/.gitignore @@ -0,0 +1 @@ +*.md diff --git a/blobs/embedded.go b/blobs/embedded.go new file mode 100644 index 00000000..808fe028 --- /dev/null +++ b/blobs/embedded.go @@ -0,0 +1,46 @@ +package blobs + +import ( + "embed" + "strings" +) + +const ( + // for micromamba upgrade, change following constants to match + // and also remember to update assets/micromamba_version.txt to match this + MicromambaVersionLimit = 1_005_008 +) + +//go:embed assets/*.yaml docs/*.md +//go:embed assets/*.zip assets/man/*.txt +//go:embed assets/*.txt +//go:embed assets/*.py +var content embed.FS + +func Asset(name string) ([]byte, error) { + return content.ReadFile(name) +} + +func MustAsset(name string) []byte { + body, err := Asset(name) + if err != nil { + panic(err) + } + return body +} + +func MustMicromamba() []byte { + body, err := micromamba.ReadFile(micromambaName) + if err != nil { + panic(err) + } + return body +} + +func MicromambaVersion() string { + body, err := Asset("assets/micromamba_version.txt") + if err != nil { + return "v0.0.0" + } + return strings.TrimSpace(string(body)) +} diff --git a/blobs/micromamba_darwin.go b/blobs/micromamba_darwin.go new file mode 100644 index 00000000..7b0abd4c --- /dev/null +++ b/blobs/micromamba_darwin.go @@ -0,0 +1,10 @@ +package blobs + +import ( + "embed" +) + +//go:embed assets/micromamba.darwin_amd64.gz +var micromamba embed.FS + +var micromambaName = "assets/micromamba.darwin_amd64.gz" diff --git a/blobs/micromamba_linux.go b/blobs/micromamba_linux.go new file mode 100644 index 00000000..44bf0fc3 --- /dev/null +++ b/blobs/micromamba_linux.go @@ -0,0 +1,10 @@ +package blobs + +import ( + "embed" +) + +//go:embed assets/micromamba.linux_amd64.gz +var micromamba embed.FS + +var micromambaName = "assets/micromamba.linux_amd64.gz" diff --git a/blobs/micromamba_windows.go b/blobs/micromamba_windows.go new file mode 100644 index 00000000..3a55328d --- /dev/null +++ b/blobs/micromamba_windows.go @@ -0,0 +1,10 @@ +package blobs + +import ( + "embed" +) + +//go:embed assets/micromamba.windows_amd64.gz +var micromamba embed.FS + +var micromambaName = "assets/micromamba.windows_amd64.gz" diff --git a/cloud/client.go b/cloud/client.go index 35efa961..8ea13144 100644 --- a/cloud/client.go +++ b/cloud/client.go @@ -1,21 +1,27 @@ package cloud import ( - "errors" + "crypto/sha256" "fmt" "io" - "io/ioutil" "net/http" + "net/url" + "os" "strings" "time" "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/pathlib" + "github.com/robocorp/rcc/set" + "github.com/robocorp/rcc/settings" "github.com/robocorp/rcc/xviper" ) type internalClient struct { endpoint string client *http.Client + tracing bool + critical bool } type Request struct { @@ -31,26 +37,45 @@ type Response struct { Status int Err error Body []byte - Elapsed time.Duration + Elapsed common.Duration } type Client interface { Endpoint() string NewRequest(string) *Request + Head(request *Request) *Response Get(request *Request) *Response Post(request *Request) *Response Put(request *Request) *Response Delete(request *Request) *Response NewClient(endpoint string) (Client, error) + WithTimeout(time.Duration) Client + WithTracing() Client + Uncritical() Client } func EnsureHttps(endpoint string) (string, error) { nice := strings.TrimRight(strings.TrimSpace(endpoint), "/") - if strings.HasPrefix(nice, "https://") { + parsed, err := url.Parse(nice) + if err != nil { + return "", err + } + if parsed.Host == "127.0.0.1" || strings.HasPrefix(parsed.Host, "127.0.0.1:") { return nice, nil } - message := fmt.Sprintf("Endpoint '%s' must start with https:// prefix.", nice) - return "", errors.New(message) + if parsed.Scheme != "https" { + return "", fmt.Errorf("Endpoint '%s' must start with https:// prefix.", nice) + } + return nice, nil +} + +func NewUnsafeClient(endpoint string) (Client, error) { + return &internalClient{ + endpoint: endpoint, + client: &http.Client{Transport: settings.Global.ConfiguredHttpTransport()}, + tracing: false, + critical: true, + }, nil } func NewClient(endpoint string) (Client, error) { @@ -60,10 +85,38 @@ func NewClient(endpoint string) (Client, error) { } return &internalClient{ endpoint: https, - client: &http.Client{}, + client: &http.Client{Transport: settings.Global.ConfiguredHttpTransport()}, + tracing: false, + critical: true, }, nil } +func (it *internalClient) Uncritical() Client { + it.critical = false + return it +} + +func (it *internalClient) WithTimeout(timeout time.Duration) Client { + return &internalClient{ + endpoint: it.endpoint, + client: &http.Client{ + Transport: settings.Global.ConfiguredHttpTransport(), + Timeout: timeout, + }, + tracing: it.tracing, + critical: it.critical, + } +} + +func (it *internalClient) WithTracing() Client { + return &internalClient{ + endpoint: it.endpoint, + client: it.client, + tracing: true, + critical: it.critical, + } +} + func (it *internalClient) NewClient(endpoint string) (Client, error) { return NewClient(endpoint) } @@ -73,12 +126,14 @@ func (it *internalClient) Endpoint() string { } func (it *internalClient) does(method string, request *Request) *Response { + stopwatch := common.Stopwatch("stopwatch") response := new(Response) - started := time.Now() + url := it.Endpoint() + request.Url + common.Trace("Doing %s %s", method, url) defer func() { - response.Elapsed = time.Now().Sub(started) + response.Elapsed = stopwatch.Elapsed() + common.Trace("%s %s took %s", method, url, response.Elapsed) }() - url := it.Endpoint() + request.Url httpRequest, err := http.NewRequest(method, url, request.Body) if err != nil { response.Status = 9001 @@ -92,24 +147,36 @@ func (it *internalClient) does(method string, request *Request) *Response { httpRequest.TransferEncoding = []string{request.TransferEncoding} } httpRequest.Header.Add("robocorp-installation-id", xviper.TrackingIdentity()) + httpRequest.Header.Add("User-Agent", common.UserAgent()) for name, value := range request.Headers { httpRequest.Header.Add(name, value) } httpResponse, err := it.client.Do(httpRequest) if err != nil { - common.Error("http.Do", err) + if it.critical { + common.Error("http.Do", err) + } else { + common.Uncritical("http.Do", err) + } response.Status = 9002 response.Err = err return response } defer httpResponse.Body.Close() + if it.tracing { + common.Trace("Response %d headers:", httpResponse.StatusCode) + keys := set.Keys(httpResponse.Header) + for _, key := range keys { + common.Trace("> %s: %q", key, httpResponse.Header[key]) + } + } response.Status = httpResponse.StatusCode if request.Stream != nil { io.Copy(request.Stream, httpResponse.Body) } else { - response.Body, response.Err = ioutil.ReadAll(httpResponse.Body) + response.Body, response.Err = io.ReadAll(httpResponse.Body) } - if common.DebugFlag { + if common.DebugFlag() { body := "ignore" if response.Status > 399 { body = string(response.Body) @@ -126,6 +193,10 @@ func (it *internalClient) NewRequest(url string) *Request { } } +func (it *internalClient) Head(request *Request) *Response { + return it.does("HEAD", request) +} + func (it *internalClient) Get(request *Request) *Response { return it.does("GET", request) } @@ -141,3 +212,56 @@ func (it *internalClient) Put(request *Request) *Response { func (it *internalClient) Delete(request *Request) *Response { return it.does("DELETE", request) } + +func Download(url, filename string) error { + common.Timeline("start %s download", filename) + defer common.Timeline("done %s download", filename) + + if pathlib.Exists(filename) { + err := os.Remove(filename) + if err != nil { + return err + } + } + + client := &http.Client{Transport: settings.Global.ConfiguredHttpTransport()} + request, err := http.NewRequest("GET", url, nil) + if err != nil { + return err + } + request.Header.Add("Accept", "application/octet-stream") + response, err := client.Do(request) + if err != nil { + return err + } + defer response.Body.Close() + + if response.StatusCode < 200 || response.StatusCode >= 300 { + return fmt.Errorf("Downloading %q failed, reason: %q!", url, response.Status) + } + + out, err := pathlib.Create(filename) + if err != nil { + return err + } + defer out.Close() + + digest := sha256.New() + many := io.MultiWriter(out, digest) + + common.Debug("Downloading %s <%s> -> %s", url, response.Status, filename) + + bytecount, err := io.Copy(many, response.Body) + if err != nil { + return err + } + + common.Timeline("downloaded %d bytes to %s", bytecount, filename) + + err = out.Sync() + if err != nil { + return err + } + + return common.Debug("%q SHA256 sum: %02x", filename, digest.Sum(nil)) +} diff --git a/cloud/client_test.go b/cloud/client_test.go index 4929b57a..c979b898 100644 --- a/cloud/client_test.go +++ b/cloud/client_test.go @@ -29,3 +29,20 @@ func TestCanCreateClient(t *testing.T) { wont_be.Nil(sut) must_be.Nil(err) } + +func TestCanEnsureHttps(t *testing.T) { + must_be, wont_be := hamlet.Specifications(t) + + _, err := cloud.EnsureHttps("http://some.server.com/endpoint") + wont_be.Nil(err) + + incoming := "https://some.server.com/endpoint" + output, err := cloud.EnsureHttps(incoming) + must_be.Nil(err) + must_be.Equal(incoming, output) + + special := "http://127.0.0.1:8192/endpoint" + output, err = cloud.EnsureHttps(special) + must_be.Nil(err) + must_be.Equal(special, output) +} diff --git a/cloud/metrics.go b/cloud/metrics.go new file mode 100644 index 00000000..057903a1 --- /dev/null +++ b/cloud/metrics.go @@ -0,0 +1,160 @@ +package cloud + +import ( + "bytes" + "encoding/json" + "fmt" + "net/url" + "os" + "runtime" + "sync" + "time" + + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/fail" + "github.com/robocorp/rcc/settings" + "github.com/robocorp/rcc/xviper" +) + +var ( + telemetryBarrier = sync.WaitGroup{} +) + +const ( + trackingUrl = `/metric-v1/%v/%v/%v/%v/%v` + batchUrl = `/metric-v1/batch` + contentType = `content-type` + applicationJson = `application/json` +) + +type ( + batchStatus struct { + File string `json:"file"` + Host string `json:"host"` + Code int `json:"code"` + Warning string `json:"warning"` + Content string `json:"content"` + } +) + +func sendMetric(metricsHost, kind, name, value string) { + common.Timeline("%s:%s = %s", kind, name, value) + defer func() { + status := recover() + if status != nil { + common.Debug("Telemetry panic recovered: %v", status) + } + telemetryBarrier.Done() + }() + client, err := NewClient(metricsHost) + if err != nil { + common.Debug("WARNING: %v (not critical)", err) + return + } + timeout := 5 * time.Second + client = client.Uncritical().WithTimeout(timeout) + timestamp := time.Now().UnixNano() + url := fmt.Sprintf(trackingUrl, url.PathEscape(kind), timestamp, url.PathEscape(xviper.TrackingIdentity()), url.PathEscape(name), url.PathEscape(value)) + common.Debug("Sending metric (timeout %v) as %v%v", timeout, metricsHost, url) + client.Put(client.NewRequest(url)) +} + +func BackgroundMetric(kind, name, value string) { + if common.WarrantyVoided() { + return + } + metricsHost := settings.Global.TelemetryURL() + if len(metricsHost) == 0 { + return + } + common.Debug("BackgroundMetric kind:%v name:%v value:%v send:%v", kind, name, value, xviper.CanTrack()) + if xviper.CanTrack() { + telemetryBarrier.Add(1) + go sendMetric(metricsHost, kind, name, value) + runtime.Gosched() + } +} + +func InternalBackgroundMetric(kind, name, value string) { + if common.Product.AllowInternalMetrics() { + BackgroundMetric(kind, name, value) + } +} + +func stdoutDump(origin error, message any) (err error) { + defer fail.Around(&err) + + body, failure := json.MarshalIndent(message, "", " ") + fail.Fast(failure) + + os.Stdout.Write(append(body, '\n')) + + return origin +} + +func BatchMetric(filename string) error { + status := &batchStatus{ + File: filename, + Code: 999, + } + + metricsHost := settings.Global.TelemetryURL() + if len(metricsHost) < 8 { + status.Warning = "No metrics host." + return stdoutDump(nil, status) + } + + blob, err := os.ReadFile(filename) + if err != nil { + status.Code = 998 + status.Warning = err.Error() + return stdoutDump(err, status) + } + + status.Host = metricsHost + + client, err := NewClient(metricsHost) + if err != nil { + status.Code = 997 + status.Warning = err.Error() + return stdoutDump(err, status) + } + + timeout := 10 * time.Second + client = client.Uncritical().WithTimeout(timeout) + request := client.NewRequest(batchUrl) + request.Headers[contentType] = applicationJson + request.Body = bytes.NewBuffer([]byte(blob)) + response := client.Put(request) + switch { + case response == nil: + status.Code = 996 + status.Warning = "Response was " + case response != nil && response.Status == 202 && response.Err == nil: + status.Code = response.Status + status.Warning = "ok" + status.Content = string(response.Body) + case response != nil && response.Err != nil: + status.Code = response.Status + status.Warning = fmt.Sprintf("Failed PUT to %s%s, reason: %v", metricsHost, batchUrl, response.Err) + status.Content = string(response.Body) + case response != nil && response.Status != 202: + status.Code = response.Status + status.Warning = fmt.Sprintf("Failed PUT to %s%s. See content for details.", metricsHost, batchUrl) + status.Content = string(response.Body) + default: + status.Code = response.Status + status.Warning = "N/A" + status.Content = string(response.Body) + } + return stdoutDump(nil, status) +} + +func WaitTelemetry() { + defer common.Timeline("wait telemetry done") + + common.Debug("wait telemetry to complete") + runtime.Gosched() + telemetryBarrier.Wait() + common.Debug("telemetry sending completed") +} diff --git a/cloud/readfile.go b/cloud/readfile.go new file mode 100644 index 00000000..91556f9d --- /dev/null +++ b/cloud/readfile.go @@ -0,0 +1,31 @@ +package cloud + +import ( + "fmt" + "net/url" + "os" + "path/filepath" + + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/pathlib" +) + +func ReadFile(resource string) ([]byte, error) { + if pathlib.IsFile(resource) { + return os.ReadFile(resource) + } + link, err := url.ParseRequestURI(resource) + if err != nil { + return os.ReadFile(resource) + } + if link.Scheme == "file" || link.Scheme == "" || pathlib.IsFile(link.Path) { + return os.ReadFile(link.Path) + } + tempfile := filepath.Join(pathlib.TempDir(), fmt.Sprintf("temp%x.part", common.When)) + defer os.Remove(tempfile) + err = Download(resource, tempfile) + if err != nil { + return nil, err + } + return os.ReadFile(tempfile) +} diff --git a/cmd/assistant.go b/cmd/assistant.go index 1b80ef9f..505efe77 100644 --- a/cmd/assistant.go +++ b/cmd/assistant.go @@ -1,6 +1,9 @@ package cmd import ( + "fmt" + + "github.com/robocorp/rcc/common" "github.com/spf13/cobra" ) @@ -8,12 +11,14 @@ var assistantCmd = &cobra.Command{ Use: "assistant", Aliases: []string{"assist", "a"}, Short: "Group of commands related to `robot assistant`.", - Long: `This set of commands relate to Robocorp Robot Assistant related tasks. -They are either local, or in relation to Robocorp Cloud and Robocorp App.`, + Long: fmt.Sprintf(`This set of commands relate to %s Robot Assistant related tasks. +They are either local, or in relation to %s Control Room and tooling.`, common.Product.Name(), common.Product.Name()), } func init() { - rootCmd.AddCommand(assistantCmd) + if common.Product.IsLegacy() { + rootCmd.AddCommand(assistantCmd) - assistantCmd.PersistentFlags().StringVarP(&accountName, "account", "", "", "Account used for Robocorp Cloud operations.") + assistantCmd.PersistentFlags().StringVarP(&accountName, "account", "", "", fmt.Sprintf("Account used for %s Control Room operations.", common.Product.Name())) + } } diff --git a/cmd/assistantList.go b/cmd/assistantList.go index 252d8e3e..efc37e94 100644 --- a/cmd/assistantList.go +++ b/cmd/assistantList.go @@ -17,12 +17,12 @@ var assistantListCmd = &cobra.Command{ Short: "Robot Assistant listing", Long: "Robot Assistant listing.", Run: func(cmd *cobra.Command, args []string) { - if common.DebugFlag { + if common.DebugFlag() { defer common.Stopwatch("Robot Assistant list query lasted").Report() } account := operations.AccountByName(AccountName()) if account == nil { - pretty.Exit(1, "Could not find account by name: %v", AccountName()) + pretty.Exit(1, "Could not find account by name: %q", AccountName()) } client, err := cloud.NewClient(account.Endpoint) if err != nil { diff --git a/cmd/assistantRun.go b/cmd/assistantRun.go index b14434a3..dcb711c5 100644 --- a/cmd/assistantRun.go +++ b/cmd/assistantRun.go @@ -9,11 +9,11 @@ import ( "github.com/robocorp/rcc/cloud" "github.com/robocorp/rcc/common" "github.com/robocorp/rcc/conda" + "github.com/robocorp/rcc/journal" "github.com/robocorp/rcc/operations" "github.com/robocorp/rcc/pathlib" "github.com/robocorp/rcc/pretty" "github.com/robocorp/rcc/robot" - "github.com/robocorp/rcc/xviper" "github.com/spf13/cobra" ) @@ -24,33 +24,33 @@ var assistantRunCmd = &cobra.Command{ Short: "Robot Assistant run", Long: "Robot Assistant run.", Run: func(cmd *cobra.Command, args []string) { + common.Timeline("cmd/assistant run entered") + defer conda.RemoveCurrentTemp() + defer journal.BuildEventStats("assistant") var status, reason string status, reason = "ERROR", "UNKNOWN" elapser := common.Stopwatch("Robot Assistant startup lasted") - if common.DebugFlag { + if common.DebugFlag() { defer common.Stopwatch("Robot Assistant run lasted").Report() } - now := time.Now() - marker := now.Unix() - ok := conda.MustConda() - if !ok { - pretty.Exit(2, "Could not get miniconda installed.") - } - defer xviper.RunMinutes().Done() account := operations.AccountByName(AccountName()) if account == nil { - pretty.Exit(1, "Could not find account by name: %v", AccountName()) + pretty.Exit(1, "Could not find account by name: %q", AccountName()) } + common.Timeline("new cloud client to %q", account.Endpoint) client, err := cloud.NewClient(account.Endpoint) if err != nil { pretty.Exit(2, "Could not create client for endpoint: %v, reason: %v", account.Endpoint, err) } + common.Timeline("new cloud client created") reason = "START_FAILURE" - operations.BackgroundMetric("rcc", "rcc.assistant.run.start", elapser.String()) + cloud.InternalBackgroundMetric(common.ControllerIdentity(), "rcc.assistant.run.start", elapser.Elapsed().String()) defer func() { - operations.BackgroundMetric("rcc", "rcc.assistant.run.stop", reason) + cloud.InternalBackgroundMetric(common.ControllerIdentity(), "rcc.assistant.run.stop", reason) }() + common.Timeline("start assistant run cloud call started") assistant, err := operations.StartAssistantRun(client, account, workspaceId, assistantId) + common.Timeline("start assistant run cloud call completed") if err != nil { pretty.Exit(3, "Could not run assistant, reason: %v", err) } @@ -67,32 +67,38 @@ var assistantRunCmd = &cobra.Command{ common.Debug("Robot Assistant run-id is %v.", assistant.RunId) common.Debug("With task '%v' from zip %v.", assistant.TaskName, assistant.Zipfile) sentinelTime := time.Now() - workarea := filepath.Join(os.TempDir(), fmt.Sprintf("workarea%x", marker)) + workarea := filepath.Join(pathlib.TempDir(), fmt.Sprintf("workarea%x", common.When)) defer os.RemoveAll(workarea) common.Debug("Using temporary workarea: %v", workarea) reason = "UNZIP_FAILURE" - err = operations.Unzip(workarea, assistant.Zipfile, false, true) + err = operations.Unzip(workarea, assistant.Zipfile, false, true, true) if err != nil { pretty.Exit(4, "Error: %v", err) } reason = "SETUP_FAILURE" targetRobot := robot.DetectConfigurationName(workarea) simple, config, todo, label := operations.LoadTaskWithEnvironment(targetRobot, assistant.TaskName, forceFlag) - artifactDir := config.ArtifactDirectory("") + artifactDir := config.ArtifactDirectory() if len(copyDirectory) > 0 && len(artifactDir) > 0 { err := os.MkdirAll(copyDirectory, 0o755) if err == nil { defer pathlib.Walk(artifactDir, pathlib.IgnoreOlder(sentinelTime).Ignore, TargetDir(copyDirectory).OverwriteBack) } } - if common.DebugFlag { + if common.DebugFlag() { elapser.Report() } defer func() { - operations.BackgroundMetric("rcc", "rcc.assistant.run.timeline.uploaded", elapser.String()) + cloud.InternalBackgroundMetric(common.ControllerIdentity(), "rcc.assistant.run.timeline.uploaded", elapser.Elapsed().String()) }() defer func() { + if len(assistant.ArtifactURL) == 0 { + pretty.Note("Pushing artifacts to Cloud skipped (disabled, no artifact URL given).") + common.Timeline("skipping publishing artifacts (disabled, no artifact URL given)") + return + } + common.Timeline("publish artifacts") publisher := operations.ArtifactPublisher{ Client: client, ArtifactPostURL: assistant.ArtifactURL, @@ -106,12 +112,12 @@ var assistantRunCmd = &cobra.Command{ } }() - operations.BackgroundMetric("rcc", "rcc.assistant.run.timeline.setup", elapser.String()) + cloud.InternalBackgroundMetric(common.ControllerIdentity(), "rcc.assistant.run.timeline.setup", elapser.Elapsed().String()) defer func() { - operations.BackgroundMetric("rcc", "rcc.assistant.run.timeline.executed", elapser.String()) + cloud.InternalBackgroundMetric(common.ControllerIdentity(), "rcc.assistant.run.timeline.executed", elapser.Elapsed().String()) }() reason = "ROBOT_FAILURE" - operations.SelectExecutionModel(captureRunFlags(), simple, todo.Commandline(), config, todo, label, false, assistant.Environment) + operations.SelectExecutionModel(captureRunFlags(true), simple, todo.Commandline(), config, todo, label, false, assistant.Environment) pretty.Ok() status, reason = "OK", "PASS" }, @@ -124,4 +130,5 @@ func init() { assistantRunCmd.Flags().StringVarP(&assistantId, "assistant", "a", "", "Assistant id to execute.") assistantRunCmd.MarkFlagRequired("assistant") assistantRunCmd.Flags().StringVarP(©Directory, "copy", "c", "", "Location to copy changed artifacts from run (optional).") + assistantRunCmd.Flags().StringVarP(&common.HolotreeSpace, "space", "s", "user", "Client specific name to identify this environment.") } diff --git a/cmd/authorize.go b/cmd/authorize.go index 4aec1c73..c2f00a4f 100644 --- a/cmd/authorize.go +++ b/cmd/authorize.go @@ -15,16 +15,21 @@ var authorizeCmd = &cobra.Command{ Short: "Convert an API key to a valid authorization JWT token.", Long: "Convert an API key to a valid authorization JWT token.", Run: func(cmd *cobra.Command, args []string) { - if common.DebugFlag { + if common.DebugFlag() { defer common.Stopwatch("Authorize query lasted").Report() } + period := &operations.TokenPeriod{ + ValidityTime: validityTime, + GracePeriod: gracePeriod, + } + period.EnforceGracePeriod() var claims *operations.Claims if granularity == "user" { - claims = operations.WorkspaceTreeClaims(validityTime * 60) + claims = operations.ViewWorkspacesClaims(period.RequestSeconds()) } else { - claims = operations.RunClaims(validityTime*60, workspaceId) + claims = operations.RunRobotClaims(period.RequestSeconds(), workspaceId) } - data, err := operations.AuthorizeClaims(AccountName(), claims) + data, err := operations.AuthorizeClaims(AccountName(), claims, period) if err != nil { pretty.Exit(3, "Error: %v", err) } @@ -38,7 +43,8 @@ var authorizeCmd = &cobra.Command{ func init() { cloudCmd.AddCommand(authorizeCmd) - authorizeCmd.Flags().IntVarP(&validityTime, "minutes", "m", 0, "How many minutes the authorization should be valid for.") + authorizeCmd.Flags().IntVarP(&validityTime, "minutes", "m", 15, "How many minutes the authorization should be valid for (minimum 15 minutes).") + authorizeCmd.Flags().IntVarP(&gracePeriod, "graceperiod", "", 5, "What is grace period buffer in minutes on top of validity minutes (minimum 5 minutes).") authorizeCmd.Flags().StringVarP(&granularity, "granularity", "g", "", "Authorization granularity (user/workspace) used in.") authorizeCmd.Flags().StringVarP(&workspaceId, "workspace", "w", "", "Workspace id to use with this command.") } diff --git a/cmd/carrier.go b/cmd/carrier.go new file mode 100644 index 00000000..78dac85b --- /dev/null +++ b/cmd/carrier.go @@ -0,0 +1,99 @@ +package cmd + +import ( + "fmt" + "os" + "path/filepath" + "time" + + "github.com/robocorp/rcc/cloud" + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/operations" + "github.com/robocorp/rcc/pathlib" + "github.com/robocorp/rcc/pretty" + "github.com/robocorp/rcc/robot" + + "github.com/spf13/cobra" +) + +var ( + carrierFile string + carrierBuild bool + carrierRun bool +) + +func buildCarrier() error { + err := operations.SelfCopy(carrierFile) + if err != nil { + return err + } + return operations.SelfAppend(carrierFile, zipfile) +} + +func runCarrier() error { + ok, err := operations.IsCarrier() + if err != nil { + return err + } + if !ok { + return fmt.Errorf("This executable is not carrier!") + } + if common.DebugFlag() { + defer common.Stopwatch("Task testrun lasted").Report() + } + now := time.Now() + testrunDir := filepath.Join(".", now.Format("2006-01-02_15_04_05")) + err = os.MkdirAll(testrunDir, 0o755) + if err != nil { + return err + } + sentinelTime := time.Now() + workarea := filepath.Join(pathlib.TempDir(), fmt.Sprintf("workarea%x", common.When)) + defer os.RemoveAll(workarea) + common.Debug("Using temporary workarea: %v", workarea) + carrier, err := operations.FindExecutable() + if err != nil { + return err + } + err = operations.CarrierUnzip(workarea, carrier, false, true) + if err != nil { + return err + } + defer pathlib.Walk(workarea, pathlib.IgnoreOlder(sentinelTime).Ignore, TargetDir(testrunDir).CopyBack) + targetRobot := robot.DetectConfigurationName(workarea) + simple, config, todo, label := operations.LoadTaskWithEnvironment(targetRobot, runTask, forceFlag) + defer common.Log("Moving outputs to %v directory.", testrunDir) + cloud.InternalBackgroundMetric(common.ControllerIdentity(), "rcc.cli.testrun", common.Version) + operations.SelectExecutionModel(captureRunFlags(false), simple, todo.Commandline(), config, todo, label, false, nil) + return nil +} + +var carrierCmd = &cobra.Command{ + Use: "carrier", + Short: "Create carrier rcc with payload.", + Long: "Create carrier rcc with payload.", + Run: func(cmd *cobra.Command, args []string) { + defer common.Stopwatch("rcc carrier lasted").Report() + if carrierBuild { + err := buildCarrier() + if err != nil { + pretty.Exit(1, "Error: %v", err) + } + pretty.Ok() + } + if carrierRun { + err := runCarrier() + if err != nil { + pretty.Exit(1, "Error: %v", err) + } + } + }, +} + +func init() { + internalCmd.AddCommand(carrierCmd) + carrierCmd.Flags().StringVarP(&zipfile, "zipfile", "z", "robot.zip", "The filename for the carrier payload.") + carrierCmd.Flags().StringVarP(&carrierFile, "carrier", "c", "carrier.exe", "The filename for the resulting carrier executable.") + carrierCmd.Flags().BoolVarP(&carrierBuild, "build", "b", false, "Build actual carrier executable.") + carrierCmd.Flags().BoolVarP(&carrierRun, "run", "r", false, "Run this executable as robot carrier.") +} diff --git a/cmd/check.go b/cmd/check.go deleted file mode 100644 index a6733425..00000000 --- a/cmd/check.go +++ /dev/null @@ -1,45 +0,0 @@ -package cmd - -import ( - "github.com/robocorp/rcc/common" - "github.com/robocorp/rcc/conda" - "github.com/robocorp/rcc/pretty" - - "github.com/spf13/cobra" -) - -var checkCmd = &cobra.Command{ - Use: "check", - Aliases: []string{"c"}, - Short: "Check if conda is installed in managed location.", - Long: `Check if conda is installed. And optionally also force download and install -conda using "rcc conda download" and "rcc conda install" commands. `, - Run: func(cmd *cobra.Command, args []string) { - if common.DebugFlag { - defer common.Stopwatch("Conda check took").Report() - } - if conda.HasConda() { - pretty.Exit(0, "OK.") - } - common.Debug("Conda is missing ...") - if !autoInstall { - pretty.Exit(1, "Error: No conda.") - } - common.Debug("Starting conda download ...") - if !(conda.DoDownload() || conda.DoDownload() || conda.DoDownload()) { - pretty.Exit(2, "Error: Conda download failed.") - } - common.Debug("Starting conda install ...") - if !conda.DoInstall() { - pretty.Exit(3, "Error: Conda install failed.") - } - common.Debug("Conda install completed ...") - pretty.Ok() - }, -} - -func init() { - condaCmd.AddCommand(checkCmd) - - checkCmd.Flags().BoolVarP(&autoInstall, "install", "i", false, "If conda is missing, download and install it automatically.") -} diff --git a/cmd/cleanup.go b/cmd/cleanup.go index 20b708ac..b837d651 100644 --- a/cmd/cleanup.go +++ b/cmd/cleanup.go @@ -9,8 +9,13 @@ import ( ) var ( - allFlag bool - daysOption int + allFlag bool + quickFlag bool + cachesFlag bool + micromambaFlag bool + downloadsFlag bool + noCompressFlag bool + daysOption int ) var cleanupCmd = &cobra.Command{ @@ -19,10 +24,10 @@ var cleanupCmd = &cobra.Command{ Long: `Cleanup removes old virtual environments from existence. After cleanup, they will not be available anymore.`, Run: func(cmd *cobra.Command, args []string) { - if common.DebugFlag { + if common.DebugFlag() { defer common.Stopwatch("Env cleanup lasted").Report() } - err := conda.Cleanup(daysOption, dryFlag, allFlag) + err := conda.Cleanup(daysOption, dryFlag, quickFlag, allFlag, micromambaFlag, downloadsFlag, noCompressFlag, cachesFlag) if err != nil { pretty.Exit(1, "Error: %v", err) } @@ -31,8 +36,13 @@ After cleanup, they will not be available anymore.`, } func init() { - envCmd.AddCommand(cleanupCmd) + configureCmd.AddCommand(cleanupCmd) cleanupCmd.Flags().BoolVarP(&dryFlag, "dryrun", "d", false, "Don't delete environments, just show what would happen.") - cleanupCmd.Flags().BoolVarP(&allFlag, "all", "a", false, "Cleanup all enviroments.") - cleanupCmd.Flags().IntVarP(&daysOption, "days", "", 30, "What is the limit in days to keep environments for (deletes environments older than this).") + cleanupCmd.Flags().BoolVarP(&cachesFlag, "caches", "", false, "Just delete all caches (hololib/conda/uv/pip) but not holotree spaces. DANGEROUS! Do not use, unless you know what you are doing.") + cleanupCmd.Flags().BoolVarP(µmambaFlag, "micromamba", "", false, "Remove micromamba installation.") + cleanupCmd.Flags().BoolVarP(&allFlag, "all", "", false, "Cleanup all enviroments.") + cleanupCmd.Flags().BoolVarP(&quickFlag, "quick", "q", false, "Cleanup most of enviroments, but leave hololib and pkgs cache intact.") + cleanupCmd.Flags().BoolVarP(&downloadsFlag, "downloads", "", false, "Cleanup downloaded cache files (pip/conda/templates)") + cleanupCmd.Flags().BoolVarP(&noCompressFlag, "no-compress", "", false, "Do not use compression in hololib content. Experimental! DANGEROUS! Do not use, unless you know what you are doing.") + cleanupCmd.Flags().IntVarP(&daysOption, "days", "", 30, "What is the limit in days to keep temp folders (deletes directories older than this).") } diff --git a/cmd/clone.go b/cmd/clone.go deleted file mode 100644 index be6a2a5f..00000000 --- a/cmd/clone.go +++ /dev/null @@ -1,34 +0,0 @@ -package cmd - -import ( - "github.com/robocorp/rcc/common" - "github.com/robocorp/rcc/conda" - "github.com/robocorp/rcc/pretty" - - "github.com/spf13/cobra" -) - -var cloneCmd = &cobra.Command{ - Use: "clone", - Short: "Debug tool for cloning folders.", - Long: `Internal debug tool for checking cloning speed in various disk drives.`, - Run: func(cmd *cobra.Command, args []string) { - source := cmd.LocalFlags().Lookup("source").Value.String() - target := cmd.LocalFlags().Lookup("target").Value.String() - defer common.Stopwatch("rcc internal clone lasted").Report() - success := conda.CloneFromTo(source, target) - if !success { - pretty.Exit(1, "Error: Cloning failed.") - } - pretty.Exit(0, "Was successful: %v", success) - }, -} - -func init() { - internalCmd.AddCommand(cloneCmd) - - cloneCmd.Flags().StringP("source", "s", "", "Source directory to clone.") - cloneCmd.Flags().StringP("target", "t", "", "Source directory to clone.") - cloneCmd.MarkFlagRequired("source") - cloneCmd.MarkFlagRequired("target") -} diff --git a/cmd/cloud.go b/cmd/cloud.go index 2eda781f..a7bd56a8 100644 --- a/cmd/cloud.go +++ b/cmd/cloud.go @@ -1,18 +1,21 @@ package cmd import ( + "fmt" + + "github.com/robocorp/rcc/common" "github.com/spf13/cobra" ) var cloudCmd = &cobra.Command{ Use: "cloud", Aliases: []string{"robocorp", "c"}, - Short: "Group of commands related to `Robocorp Cloud`.", - Long: `This group of commands apply to communication with Robocorp Cloud.`, + Short: fmt.Sprintf("Group of commands related to `%s Control Room`.", common.Product.Name()), + Long: fmt.Sprintf(`This group of commands apply to communication with %s Control Room.`, common.Product.Name()), } func init() { rootCmd.AddCommand(cloudCmd) - cloudCmd.PersistentFlags().StringVarP(&accountName, "account", "a", "", "Account used for Robocorp Cloud operations.") + cloudCmd.PersistentFlags().StringVarP(&accountName, "account", "a", "", fmt.Sprintf("Account used for %s Control Room operations.", common.Product.Name())) } diff --git a/cmd/cloudNew.go b/cmd/cloudNew.go index 52826e48..4dc615b5 100644 --- a/cmd/cloudNew.go +++ b/cmd/cloudNew.go @@ -1,6 +1,10 @@ package cmd import ( + "encoding/json" + "fmt" + "os" + "github.com/robocorp/rcc/cloud" "github.com/robocorp/rcc/common" "github.com/robocorp/rcc/operations" @@ -11,15 +15,15 @@ import ( var newCloudCmd = &cobra.Command{ Use: "new", - Short: "Create a new robot into Robocorp Cloud.", - Long: "Create a new robot into Robocorp Cloud.", + Short: fmt.Sprintf("Create a new robot into %s Control Room.", common.Product.Name()), + Long: fmt.Sprintf("Create a new robot into %s Control Room.", common.Product.Name()), Run: func(cmd *cobra.Command, args []string) { - if common.DebugFlag { + if common.DebugFlag() { defer common.Stopwatch("New robot creation lasted").Report() } account := operations.AccountByName(AccountName()) if account == nil { - pretty.Exit(1, "Could not find account by name: %v", AccountName()) + pretty.Exit(1, "Could not find account by name: %q", AccountName()) } client, err := cloud.NewClient(account.Endpoint) if err != nil { @@ -29,7 +33,13 @@ var newCloudCmd = &cobra.Command{ if err != nil { pretty.Exit(3, "Error: %v", err) } - common.Log("Created new robot named '%s' with identity %s.", reply["name"], reply["id"]) + if jsonFlag { + result, err := json.MarshalIndent(reply, "", " ") + pretty.Guard(err == nil, 1, "Json converion failed, reason: %v", err) + fmt.Fprintf(os.Stdout, "%s\n", result) + } else { + common.Log("Created new robot named '%s' with identity %s.", reply["name"], reply["id"]) + } }, } @@ -39,4 +49,5 @@ func init() { newCloudCmd.MarkFlagRequired("robot") newCloudCmd.Flags().StringVarP(&workspaceId, "workspace", "w", "", "Workspace id to use as creation target.") newCloudCmd.MarkFlagRequired("workspace") + newCloudCmd.Flags().BoolVarP(&jsonFlag, "json", "j", false, "Output in JSON format") } diff --git a/cmd/cloudPrepare.go b/cmd/cloudPrepare.go new file mode 100644 index 00000000..4737aa28 --- /dev/null +++ b/cmd/cloudPrepare.go @@ -0,0 +1,73 @@ +package cmd + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/robocorp/rcc/cloud" + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/htfs" + "github.com/robocorp/rcc/journal" + "github.com/robocorp/rcc/operations" + "github.com/robocorp/rcc/pathlib" + "github.com/robocorp/rcc/pretty" + "github.com/robocorp/rcc/robot" + + "github.com/spf13/cobra" +) + +var prepareCloudCmd = &cobra.Command{ + Use: "prepare", + Short: "Prepare cloud robot for fast startup time in local computer.", + Long: "Prepare cloud robot for fast startup time in local computer.", + Run: func(cmd *cobra.Command, args []string) { + defer journal.BuildEventStats("prepare") + if common.DebugFlag() { + defer common.Stopwatch("Cloud prepare lasted").Report() + } + + zipfile := filepath.Join(pathlib.TempDir(), fmt.Sprintf("summon%x.zip", common.When)) + defer os.Remove(zipfile) + + workarea := filepath.Join(pathlib.TempDir(), fmt.Sprintf("workarea%x", common.When)) + defer os.RemoveAll(workarea) + + account := operations.AccountByName(AccountName()) + pretty.Guard(account != nil, 2, "Could not find account by name: %q", AccountName()) + + client, err := cloud.NewClient(account.Endpoint) + pretty.Guard(err == nil, 3, "Could not create client for endpoint: %v, reason: %v", account.Endpoint, err) + + err = operations.DownloadCommand(client, account, workspaceId, robotId, zipfile, common.DebugFlag()) + pretty.Guard(err == nil, 4, "Error: %v", err) + + common.Debug("Using temporary workarea: %v", workarea) + err = operations.Unzip(workarea, zipfile, false, true, true) + pretty.Guard(err == nil, 5, "Error: %v", err) + + robotfile, err := pathlib.FindNamedPath(workarea, "robot.yaml") + pretty.Guard(err == nil, 6, "Error: %v", err) + + config, err := robot.LoadRobotYaml(robotfile, false) + pretty.Guard(err == nil, 7, "Error: %v", err) + pretty.Guard(config.UsesConda(), 0, "Ok.") + + var label string + condafile := config.CondaConfigFile() + label, _, err = htfs.NewEnvironment(condafile, config.Holozip(), true, false, operations.PullCatalog) + pretty.Guard(err == nil, 8, "Error: %v", err) + + common.Log("Prepared %q.", label) + pretty.Ok() + }, +} + +func init() { + cloudCmd.AddCommand(prepareCloudCmd) + prepareCloudCmd.Flags().StringVarP(&workspaceId, "workspace", "w", "", "The workspace id to use as the download source.") + prepareCloudCmd.MarkFlagRequired("workspace") + prepareCloudCmd.Flags().StringVarP(&robotId, "robot", "r", "", "The robot id to use as the download source.") + prepareCloudCmd.MarkFlagRequired("robot") + prepareCloudCmd.Flags().StringVarP(&common.HolotreeSpace, "space", "s", "user", "Client specific name to identify this environment.") +} diff --git a/cmd/command_darwin.go b/cmd/command_darwin.go new file mode 100644 index 00000000..fa53a5d3 --- /dev/null +++ b/cmd/command_darwin.go @@ -0,0 +1,18 @@ +package cmd + +import ( + "os" + + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/pathlib" + "github.com/robocorp/rcc/pretty" +) + +func osSpecificHolotreeSharing(enable bool) { + if !enable { + return + } + pathlib.ForceShared() + err := os.WriteFile(common.SharedMarkerLocation(), []byte(common.Version), 0644) + pretty.Guard(err == nil, 3, "Could not write %q, reason: %v", common.SharedMarkerLocation(), err) +} diff --git a/cmd/command_linux.go b/cmd/command_linux.go new file mode 100644 index 00000000..684969b6 --- /dev/null +++ b/cmd/command_linux.go @@ -0,0 +1,24 @@ +package cmd + +import ( + "os" + "path/filepath" + + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/pathlib" + "github.com/robocorp/rcc/pretty" +) + +func osSpecificHolotreeSharing(enable bool) { + if !enable { + return + } + pathlib.ForceShared() + parent := filepath.Dir(common.Product.HoloLocation()) + _, err := pathlib.ForceSharedDir(parent) + pretty.Guard(err == nil, 1, "Could not enable shared location at %q, reason: %v", parent, err) + _, err = pathlib.ForceSharedDir(common.Product.HoloLocation()) + pretty.Guard(err == nil, 2, "Could not enable shared location at %q, reason: %v", common.Product.HoloLocation(), err) + err = os.WriteFile(common.SharedMarkerLocation(), []byte(common.Version), 0644) + pretty.Guard(err == nil, 3, "Could not write %q, reason: %v", common.SharedMarkerLocation(), err) +} diff --git a/cmd/command_windows.go b/cmd/command_windows.go new file mode 100644 index 00000000..93090b9a --- /dev/null +++ b/cmd/command_windows.go @@ -0,0 +1,26 @@ +package cmd + +import ( + "os" + "path/filepath" + + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/pathlib" + "github.com/robocorp/rcc/pretty" + "github.com/robocorp/rcc/shell" +) + +func osSpecificHolotreeSharing(enable bool) { + if !enable { + return + } + pathlib.ForceShared() + parent := filepath.Dir(common.Product.HoloLocation()) + _, err := pathlib.ForceSharedDir(parent) + pretty.Guard(err == nil, 1, "Could not enable shared location at %q, reason: %v", parent, err) + task := shell.New(nil, ".", "icacls", parent, "/grant", "*S-1-5-32-545:(OI)(CI)M", "/T", "/Q") + _, err = task.Execute(false) + pretty.Guard(err == nil, 2, "Could not set 'icacls' settings, reason: %v", err) + err = os.WriteFile(common.SharedMarkerLocation(), []byte(common.Version), 0644) + pretty.Guard(err == nil, 3, "Could not write %q, reason: %v", common.SharedMarkerLocation(), err) +} diff --git a/cmd/commontools.go b/cmd/commontools.go index 24919de2..f8e0e81d 100644 --- a/cmd/commontools.go +++ b/cmd/commontools.go @@ -8,6 +8,10 @@ const ( environmentAccount = `RCC_CREDENTIALS_ID` ) +func Has(value string) bool { + return len(value) > 0 +} + func AccountName() string { if len(accountName) > 0 { return accountName diff --git a/cmd/community.go b/cmd/community.go index e6e34c1d..46f636f4 100644 --- a/cmd/community.go +++ b/cmd/community.go @@ -1,16 +1,21 @@ package cmd import ( + "fmt" + + "github.com/robocorp/rcc/common" "github.com/spf13/cobra" ) var communityCmd = &cobra.Command{ Use: "community", Aliases: []string{"co"}, - Short: "Group of commands related to `Robocorp Community`.", + Short: fmt.Sprintf("Group of commands related to `%s Community`.", common.Product.Name()), Long: `This group of commands apply to community provided robots and services.`, } func init() { - rootCmd.AddCommand(communityCmd) + if common.Product.IsLegacy() { + rootCmd.AddCommand(communityCmd) + } } diff --git a/cmd/communitypull.go b/cmd/communitypull.go index 8dfc5dd1..4e534952 100644 --- a/cmd/communitypull.go +++ b/cmd/communitypull.go @@ -4,10 +4,10 @@ import ( "fmt" "os" "path/filepath" - "time" "github.com/robocorp/rcc/common" "github.com/robocorp/rcc/operations" + "github.com/robocorp/rcc/pathlib" "github.com/robocorp/rcc/pretty" "github.com/spf13/cobra" @@ -23,11 +23,11 @@ var communityPullCmd = &cobra.Command{ Long: "Pull a robot from URL or community sources.", Args: cobra.ExactArgs(1), Run: func(cmd *cobra.Command, args []string) { - if common.DebugFlag { + if common.DebugFlag() { defer common.Stopwatch("Pull lasted").Report() } - zipfile := filepath.Join(os.TempDir(), fmt.Sprintf("pull%x.zip", time.Now().Unix())) + zipfile := filepath.Join(pathlib.TempDir(), fmt.Sprintf("pull%x.zip", common.When)) defer os.Remove(zipfile) common.Debug("Using temporary zipfile at %v", zipfile) @@ -46,7 +46,7 @@ var communityPullCmd = &cobra.Command{ pretty.Exit(1, "Download failed: %v!", err) } - err = operations.Unzip(directory, zipfile, true, false) + err = operations.Unzip(directory, zipfile, true, false, true) if err != nil { pretty.Exit(1, "Error: %v", err) } @@ -56,8 +56,10 @@ var communityPullCmd = &cobra.Command{ } func init() { - communityCmd.AddCommand(communityPullCmd) - rootCmd.AddCommand(communityPullCmd) - communityPullCmd.Flags().StringVarP(&branch, "branch", "b", "main", "Branch/tag/commitid to use as basis for robot.") - communityPullCmd.Flags().StringVarP(&directory, "directory", "d", ".", "The root directory to extract the robot into.") + if common.Product.IsLegacy() { + communityCmd.AddCommand(communityPullCmd) + rootCmd.AddCommand(communityPullCmd) + communityPullCmd.Flags().StringVarP(&branch, "branch", "b", "main", "Branch/tag/commitid to use as basis for robot.") + communityPullCmd.Flags().StringVarP(&directory, "directory", "d", ".", "The root directory to extract the robot into.") + } } diff --git a/cmd/conda.go b/cmd/conda.go deleted file mode 100644 index b1441e63..00000000 --- a/cmd/conda.go +++ /dev/null @@ -1,15 +0,0 @@ -package cmd - -import ( - "github.com/spf13/cobra" -) - -var condaCmd = &cobra.Command{ - Use: "conda", - Short: "Group of commands related to `conda installation`.", - Long: `Conda specific funtionality captured in this set of subcommands.`, -} - -func init() { - rootCmd.AddCommand(condaCmd) -} diff --git a/cmd/condadownload.go b/cmd/condadownload.go deleted file mode 100644 index cdd40cb8..00000000 --- a/cmd/condadownload.go +++ /dev/null @@ -1,25 +0,0 @@ -package cmd - -import ( - "github.com/robocorp/rcc/conda" - "github.com/robocorp/rcc/pretty" - - "github.com/spf13/cobra" -) - -var condaDownloadCmd = &cobra.Command{ - Use: "download", - Aliases: []string{"dl", "d"}, - Short: "Download the miniconda3 installer.", - Long: `Downloads the miniconda3 installer for this platform.`, - Args: cobra.NoArgs, - Run: func(cmd *cobra.Command, args []string) { - if !(conda.DoDownload() || conda.DoDownload() || conda.DoDownload()) { - pretty.Exit(1, "Download failed.") - } - }, -} - -func init() { - condaCmd.AddCommand(condaDownloadCmd) -} diff --git a/cmd/configure.go b/cmd/configure.go index 39e14e44..470713ec 100644 --- a/cmd/configure.go +++ b/cmd/configure.go @@ -1,12 +1,15 @@ package cmd import ( + "fmt" + + "github.com/robocorp/rcc/common" "github.com/spf13/cobra" ) var configureCmd = &cobra.Command{ - Use: "configure", - Aliases: []string{"conf", "config"}, + Use: "configuration", + Aliases: []string{"conf", "config", "configure"}, Short: "Group of commands related to `rcc configuration`.", Long: "Group of commands to configure rcc with your settings.", } @@ -14,5 +17,5 @@ var configureCmd = &cobra.Command{ func init() { rootCmd.AddCommand(configureCmd) - configureCmd.PersistentFlags().StringVarP(&accountName, "account", "a", "", "Account used for Robocorp Cloud task.") + configureCmd.PersistentFlags().StringVarP(&accountName, "account", "a", "", fmt.Sprintf("Account used for %s Control Room task.", common.Product.Name())) } diff --git a/cmd/configureTLSexport.go b/cmd/configureTLSexport.go new file mode 100644 index 00000000..ecd3c9e2 --- /dev/null +++ b/cmd/configureTLSexport.go @@ -0,0 +1,66 @@ +package cmd + +import ( + "crypto/x509" + "os" + + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/fail" + "github.com/robocorp/rcc/operations" + "github.com/robocorp/rcc/pretty" + "github.com/robocorp/rcc/settings" + "github.com/spf13/cobra" +) + +var ( + pemFile string +) + +var tlsExportCmd = &cobra.Command{ + Use: "tlsexport +", + Short: "Export TLS certificates from set of secure and unsecure URLs.", + Long: `Export TLS certificates from set of secure and unsecure URLs. + +CLI examples: + rcc configuration tlsexport --pemfile export.pem robot_urls.yaml + rcc configuration tlsexport --pemfile many.pem company_urls.yaml robot_urls.yaml more_urls.yaml + + +Configuration example in YAML format: +# trusted: +# - https://api.eu1.robocorp.com/ +# - https://pypi.org/ +# - https://files.pythonhosted.org/ +# untrusted: +# - https://self-signed.badssl.com/ + +Note: exported PEM file is sorted, so it can be easily compared to other export. + Use your fabourite diffing tool there. +`, + Args: cobra.MinimumNArgs(1), + Run: func(cmd *cobra.Command, configfiles []string) { + pretty.Guard(!settings.Global.HasCaBundle(), 1, "Cannot create certificate bundle, while profile provides %q!", common.CaBundleFile()) + err := operations.TLSExport(pemFile, configfiles) + pretty.Guard(err == nil, 2, "Probe failure: %v", err) + err = certificatePool(pemFile) + pretty.Guard(err == nil, 3, "Could not import created CA bundle, reason: %v", err) + pretty.Ok() + }, +} + +func certificatePool(bundle string) (err error) { + defer fail.Around(&err) + + pool, err := x509.SystemCertPool() + fail.On(err != nil, "Could not get system certificate pool, reason: %v", err) + blob, err := os.ReadFile(bundle) + fail.On(err != nil, "Could not get read certificate bundle from %q, reason: %v", bundle, err) + fail.On(!pool.AppendCertsFromPEM(blob), "Could not add certs from %q to created pool!", bundle) + return nil +} + +func init() { + configureCmd.AddCommand(tlsExportCmd) + tlsExportCmd.Flags().StringVarP(&pemFile, "pemfile", "p", "", "Name of exported PEM file to write.") + tlsExportCmd.MarkFlagRequired("pemfile") +} diff --git a/cmd/configureTLSprobe.go b/cmd/configureTLSprobe.go new file mode 100644 index 00000000..00b46e2f --- /dev/null +++ b/cmd/configureTLSprobe.go @@ -0,0 +1,50 @@ +package cmd + +import ( + "strings" + + "github.com/robocorp/rcc/operations" + "github.com/robocorp/rcc/pretty" + "github.com/spf13/cobra" +) + +func fixHosts(hosts []string) []string { + servers := make([]string, len(hosts)) + for at, host := range hosts { + if strings.Contains(host, ":") { + servers[at] = host + } else { + servers[at] = host + ":443" + } + } + return servers +} + +var tlsProbeCmd = &cobra.Command{ + Use: "tlsprobe +", + Short: "Probe host:port combinations for supported TLS versions.", + Long: `Probe host:port combinations for supported TLS versions. + +This command will show following information on your TLS settings: +- current DNS resolution give host +- which TLS versions are available on specific host:port combo +- server name, address, port, and cipher suite that actually was negotiated +- certificate chains that was seen on that connection + +Examples: + rcc configuration tlsprobe www.bing.com www.google.com + rcc configuration tlsprobe outlook.office365.com:993 outlook.office365.com:995 + rcc configuration tlsprobe api.us1.robocorp.com api.eu1.robocorp.com +`, + Args: cobra.MinimumNArgs(1), + Run: func(cmd *cobra.Command, args []string) { + servers := fixHosts(args) + err := operations.TLSProbe(servers) + pretty.Guard(err == nil, 1, "Probe failure: %v", err) + pretty.Ok() + }, +} + +func init() { + configureCmd.AddCommand(tlsProbeCmd) +} diff --git a/cmd/configureprofile.go b/cmd/configureprofile.go new file mode 100644 index 00000000..db2d717a --- /dev/null +++ b/cmd/configureprofile.go @@ -0,0 +1,184 @@ +package cmd + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/operations" + "github.com/robocorp/rcc/pretty" + "github.com/robocorp/rcc/settings" + + "github.com/spf13/cobra" +) + +var ( + configFile string + profileName string + clearProfile bool + immediateSwitch bool +) + +func profileMap() map[string]string { + pattern := common.ExpandPath(filepath.Join(common.Product.Home(), "profile_*.yaml")) + found, err := filepath.Glob(pattern) + pretty.Guard(err == nil, 1, "Error while searching profiles: %v", err) + profiles := make(map[string]string) + for _, name := range found { + profile := settings.Profile{} + err = profile.LoadFrom(name) + if err == nil { + profiles[profile.Name] = profile.Description + } + } + return profiles +} + +func jsonListProfiles() { + profiles := make(map[string]interface{}) + profiles["profiles"] = profileMap() + profiles["current"] = settings.Global.Name() + content, err := operations.NiceJsonOutput(profiles) + pretty.Guard(err == nil, 1, "Error serializing profiles: %v", err) + common.Stdout("%s\n", content) +} + +func listProfiles() { + profiles := profileMap() + pretty.Guard(len(profiles) > 0, 2, "No profiles found, you must first import some.") + common.Stdout("Available profiles:\n") + for name, description := range profiles { + common.Stdout("- %s: %s\n", name, description) + } + common.Stdout("\n") +} + +func profileFullPath(name string) string { + filename := fmt.Sprintf("profile_%s.yaml", strings.ToLower(name)) + return common.ExpandPath(filepath.Join(common.Product.Home(), filename)) +} + +func loadNamedProfile(name string) *settings.Profile { + fullpath := profileFullPath(name) + profile := &settings.Profile{} + err := profile.LoadFrom(fullpath) + pretty.Guard(err == nil, 3, "Error while loading/parsing profile, reason: %v", err) + return profile +} + +func switchProfileTo(name string) { + profile := loadNamedProfile(name) + err := profile.Activate() + pretty.Guard(err == nil, 4, "Error while activating profile, reason: %v", err) +} + +func cleanupProfile() { + profile := settings.Profile{} + err := profile.Remove() + pretty.Guard(err == nil, 5, "Error while clearing profile, reason: %v", err) +} + +var configureSwitchCmd = &cobra.Command{ + Use: "switch", + Short: fmt.Sprintf("Switch active configuration profile for %s tooling.", common.Product.Name()), + Long: fmt.Sprintf("Switch active configuration profile for %s tooling.", common.Product.Name()), + Run: func(cmd *cobra.Command, args []string) { + if common.DebugFlag() { + defer common.Stopwatch("Configuration switch lasted").Report() + } + if clearProfile { + cleanupProfile() + pretty.Ok() + } else if len(profileName) == 0 { + if jsonFlag { + jsonListProfiles() + } else { + listProfiles() + common.Stdout("Currently active profile is: %s\n", settings.Global.Name()) + pretty.Ok() + } + } else { + switchProfileTo(profileName) + pretty.Ok() + } + }, +} + +var configureRemoveCmd = &cobra.Command{ + Use: "remove", + Short: fmt.Sprintf("Remove named a configuration profile for %s tooling.", common.Product.Name()), + Long: fmt.Sprintf("Remove named a configuration profile for %s tooling.", common.Product.Name()), + Run: func(cmd *cobra.Command, args []string) { + if common.DebugFlag() { + defer common.Stopwatch("Configuration remove lasted").Report() + } + profiles := profileMap() + description, ok := profiles[profileName] + pretty.Guard(ok, 2, "No match for profile with name %q.", profileName) + fullpath := profileFullPath(profileName) + + common.Log("Trying to remove profile: %s %q [%s].", profileName, description, fullpath) + err := os.Remove(fullpath) + pretty.Guard(err == nil, 5, "Error while removing profile file %q, reason: %v", fullpath, err) + + pretty.Ok() + }, +} + +var configureExportCmd = &cobra.Command{ + Use: "export", + Short: fmt.Sprintf("Export a configuration profile for %s tooling.", common.Product.Name()), + Long: fmt.Sprintf("Export a configuration profile for %s tooling.", common.Product.Name()), + Run: func(cmd *cobra.Command, args []string) { + if common.DebugFlag() { + defer common.Stopwatch("Configuration export lasted").Report() + } + profile := loadNamedProfile(profileName) + err := profile.SaveAs(configFile) + pretty.Guard(err == nil, 1, "Error while exporting profile, reason: %v", err) + pretty.Ok() + }, +} + +var configureImportCmd = &cobra.Command{ + Use: "import", + Short: fmt.Sprintf("Import a configuration profile for %s tooling.", common.Product.Name()), + Long: fmt.Sprintf("Import a configuration profile for %s tooling.", common.Product.Name()), + Run: func(cmd *cobra.Command, args []string) { + if common.DebugFlag() { + defer common.Stopwatch("Configuration import lasted").Report() + } + profile := &settings.Profile{} + err := profile.LoadFrom(configFile) + pretty.Guard(err == nil, 1, "Error while loading profile: %v", err) + err = profile.Import() + pretty.Guard(err == nil, 2, "Error while importing profile: %v", err) + if immediateSwitch { + switchProfileTo(profile.Name) + } + pretty.Ok() + }, +} + +func init() { + configureCmd.AddCommand(configureSwitchCmd) + configureSwitchCmd.Flags().StringVarP(&profileName, "profile", "p", "", "The name of configuration profile to activate.") + configureSwitchCmd.Flags().BoolVarP(&clearProfile, "noprofile", "n", false, "Remove active profile, and reset to defaults.") + configureSwitchCmd.Flags().BoolVarP(&jsonFlag, "json", "j", false, "Show profile list as JSON stream.") + + configureCmd.AddCommand(configureRemoveCmd) + configureRemoveCmd.Flags().StringVarP(&profileName, "profile", "p", "", "The name of configuration profile to remove.") + configureRemoveCmd.MarkFlagRequired("profile") + + configureCmd.AddCommand(configureExportCmd) + configureExportCmd.Flags().StringVarP(&configFile, "filename", "f", "exported_profile.yaml", "The filename where configuration profile is exported.") + configureExportCmd.Flags().StringVarP(&profileName, "profile", "p", "", "The name of configuration profile to export.") + configureExportCmd.MarkFlagRequired("profile") + + configureCmd.AddCommand(configureImportCmd) + configureImportCmd.Flags().BoolVarP(&immediateSwitch, "switch", "s", false, "Immediately switch to use new profile.") + configureImportCmd.Flags().StringVarP(&configFile, "filename", "f", "exported_profile.yaml", "The filename to import as configuration profile.") + configureImportCmd.MarkFlagRequired("filename") +} diff --git a/cmd/credentials.go b/cmd/credentials.go index 57b42bb9..78ac5853 100644 --- a/cmd/credentials.go +++ b/cmd/credentials.go @@ -1,12 +1,15 @@ package cmd import ( + "fmt" "strings" + "time" "github.com/robocorp/rcc/cloud" "github.com/robocorp/rcc/common" "github.com/robocorp/rcc/operations" "github.com/robocorp/rcc/pretty" + "github.com/robocorp/rcc/settings" "github.com/spf13/cobra" ) @@ -17,11 +20,11 @@ var ( var credentialsCmd = &cobra.Command{ Use: "credentials [credentials]", - Short: "Manage Robocorp Cloud API credentials.", - Long: "Manage Robocorp Cloud API credentials for later use.", + Short: fmt.Sprintf("Manage %s Control Room API credentials.", common.Product.Name()), + Long: fmt.Sprintf("Manage %s Control Room API credentials for later use.", common.Product.Name()), Args: cobra.MaximumNArgs(1), Run: func(cmd *cobra.Command, args []string) { - if common.DebugFlag { + if common.DebugFlag() { defer common.Stopwatch("Credentials query lasted").Report() } var account, credentials, endpoint string @@ -45,7 +48,7 @@ var credentialsCmd = &cobra.Command{ } endpoint = endpointUrl if len(endpoint) == 0 { - endpoint = common.DefaultEndpoint + endpoint = settings.Global.DefaultEndpoint() } https, err := cloud.EnsureHttps(endpoint) if err != nil { @@ -53,7 +56,7 @@ var credentialsCmd = &cobra.Command{ } parts := strings.Split(credentials, ":") if len(parts) != 2 { - pretty.Exit(1, "Error: No valid credentials detected. Copy them from Robocorp Cloud.") + pretty.Exit(1, "Error: No valid credentials detected. Copy them from %s Control Room.", common.Product.Name()) } common.Log("Adding credentials: %v", parts) operations.UpdateCredentials(account, https, parts[0], parts[1]) @@ -67,9 +70,9 @@ var credentialsCmd = &cobra.Command{ func localDelete(accountName string) { account := operations.AccountByName(accountName) if account == nil { - pretty.Exit(1, "Could not find account by name: %v", accountName) + pretty.Exit(1, "Could not find account by name: %q", accountName) } - err := account.Delete() + err := account.Delete(10 * time.Second) if err != nil { pretty.Exit(3, "Error: %v", err) } @@ -79,9 +82,9 @@ func localDelete(accountName string) { func init() { configureCmd.AddCommand(credentialsCmd) - credentialsCmd.Flags().BoolVarP(&deleteCredentialsFlag, "delete", "", false, "Delete this account and corresponding Cloud credentials! DANGER!") + credentialsCmd.Flags().BoolVarP(&deleteCredentialsFlag, "delete", "", false, "Delete this account and corresponding Control Room credentials! DANGER!") credentialsCmd.Flags().BoolVarP(&defaultFlag, "default", "d", false, "Set this as the default account.") credentialsCmd.Flags().BoolVarP(&jsonFlag, "json", "j", false, "Output in JSON format.") credentialsCmd.Flags().BoolVarP(&verifiedFlag, "verified", "v", false, "Updates the verified timestamp, if the credentials are still active.") - credentialsCmd.Flags().StringVarP(&endpointUrl, "endpoint", "e", "", "Robocorp Cloud endpoint used with the given account (or default).") + credentialsCmd.Flags().StringVarP(&endpointUrl, "endpoint", "e", "", fmt.Sprintf("%s Control Room endpoint used with the given account (or default).", common.Product.Name())) } diff --git a/cmd/delete.go b/cmd/delete.go deleted file mode 100644 index 989640fd..00000000 --- a/cmd/delete.go +++ /dev/null @@ -1,25 +0,0 @@ -package cmd - -import ( - "github.com/robocorp/rcc/common" - "github.com/robocorp/rcc/conda" - - "github.com/spf13/cobra" -) - -var deleteCmd = &cobra.Command{ - Use: "delete", - Short: "Delete one managed virtual environment.", - Long: `Delete the given virtual environment from existence. -After deletion, it will not be available anymore.`, - Run: func(cmd *cobra.Command, args []string) { - for _, label := range args { - common.Log("Removing %v", label) - conda.RemoveEnvironment(label) - } - }, -} - -func init() { - envCmd.AddCommand(deleteCmd) -} diff --git a/cmd/diagnostics.go b/cmd/diagnostics.go new file mode 100644 index 00000000..a9492e92 --- /dev/null +++ b/cmd/diagnostics.go @@ -0,0 +1,43 @@ +package cmd + +import ( + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/operations" + "github.com/robocorp/rcc/pretty" + + "github.com/spf13/cobra" +) + +var ( + fileOption string + robotOption string + quickFilterFlag bool +) + +var diagnosticsCmd = &cobra.Command{ + Use: "diagnostics", + Aliases: []string{"diagnostic", "diag"}, + Short: "Run system diagnostics to help resolve rcc issues.", + Long: "Run system diagnostics to help resolve rcc issues.", + Run: func(cmd *cobra.Command, args []string) { + if common.DebugFlag() { + defer common.Stopwatch("Diagnostic run lasted").Report() + } + _, err := operations.ProduceDiagnostics(fileOption, robotOption, jsonFlag, productionFlag, quickFilterFlag || common.WarrantyVoided()) + if err != nil { + pretty.Exit(1, "Error: %v", err) + } + pretty.Ok() + }, +} + +func init() { + configureCmd.AddCommand(diagnosticsCmd) + rootCmd.AddCommand(diagnosticsCmd) + + diagnosticsCmd.Flags().BoolVarP(&jsonFlag, "json", "j", false, "Output in JSON format.") + diagnosticsCmd.Flags().BoolVarP(&quickFilterFlag, "quick", "q", false, "Only run quick diagnostics.") + diagnosticsCmd.Flags().StringVarP(&fileOption, "file", "f", "", "Save output into a file.") + diagnosticsCmd.Flags().StringVarP(&robotOption, "robot", "r", "", "Full path to 'robot.yaml' configuration file. [optional]") + diagnosticsCmd.Flags().BoolVarP(&productionFlag, "production", "p", false, "Checks for production level robots. [optional]") +} diff --git a/cmd/dirhash.go b/cmd/dirhash.go index acfadecc..26208327 100644 --- a/cmd/dirhash.go +++ b/cmd/dirhash.go @@ -1,14 +1,23 @@ package cmd import ( + "fmt" "os" + "path/filepath" + "sort" "github.com/robocorp/rcc/common" "github.com/robocorp/rcc/conda" + "github.com/robocorp/rcc/pretty" "github.com/spf13/cobra" ) +var ( + showDiff bool + showIntermediateDirhashes bool +) + var dirhashCmd = &cobra.Command{ Use: "dirhash", Short: "Calculate hash for directory content.", @@ -16,6 +25,7 @@ var dirhashCmd = &cobra.Command{ Args: cobra.MinimumNArgs(1), Run: func(cmd *cobra.Command, args []string) { defer common.Stopwatch("rcc dirhash lasted").Report() + diffMaps := make([]map[string]string, 0, len(args)) for _, directory := range args { stat, err := os.Stat(directory) if err != nil { @@ -25,17 +35,45 @@ var dirhashCmd = &cobra.Command{ if !stat.IsDir() { continue } - digest, err := conda.DigestFor(directory) + fullpath, err := filepath.Abs(directory) + if err != nil { + continue + } + collector := make(map[string]string) + digest, err := conda.DigestFor(fullpath, collector) if err != nil { common.Error("dirhash", err) continue } - result := conda.Hexdigest(digest) + collector = conda.MakeRelativeMap(fullpath, collector) + diffMaps = append(diffMaps, collector) + result := common.Hexdigest(digest) common.Log("+ %v %v", result, directory) + if showIntermediateDirhashes { + relative := make(map[string]string) + keyset := make([]string, 0, len(collector)) + for key, value := range collector { + keyset = append(keyset, key) + relative[key] = value + } + sort.Strings(keyset) + for _, key := range keyset { + fmt.Printf("%s %s\n", relative[key], key) + } + fmt.Println() + } + } + if showDiff && len(diffMaps) != 2 { + pretty.Exit(1, "Diff expects exactly 2 environments, now got %d!", len(diffMaps)) + } + if showDiff { + conda.DirhashDiff(diffMaps[0], diffMaps[1], false) } }, } func init() { internalCmd.AddCommand(dirhashCmd) + dirhashCmd.Flags().BoolVarP(&showIntermediateDirhashes, "print", "", false, "Print all intermediate folder hashes also.") + dirhashCmd.Flags().BoolVarP(&showDiff, "diff", "", false, "Diff two environments with differences.") } diff --git a/cmd/download.go b/cmd/download.go index aebbfe5d..5e3ad859 100644 --- a/cmd/download.go +++ b/cmd/download.go @@ -1,6 +1,8 @@ package cmd import ( + "fmt" + "github.com/robocorp/rcc/cloud" "github.com/robocorp/rcc/common" "github.com/robocorp/rcc/operations" @@ -11,21 +13,21 @@ import ( var downloadCmd = &cobra.Command{ Use: "download", - Short: "Fetch an existing robot from Robocorp Cloud.", - Long: "Fetch an existing robot from Robocorp Cloud.", + Short: fmt.Sprintf("Fetch an existing robot from %s Control Room.", common.Product.Name()), + Long: fmt.Sprintf("Fetch an existing robot from %s Control Room.", common.Product.Name()), Run: func(cmd *cobra.Command, args []string) { - if common.DebugFlag { + if common.DebugFlag() { defer common.Stopwatch("Download lasted").Report() } account := operations.AccountByName(AccountName()) if account == nil { - pretty.Exit(1, "Could not find account by name: %v", AccountName()) + pretty.Exit(1, "Could not find account by name: %q", AccountName()) } client, err := cloud.NewClient(account.Endpoint) if err != nil { pretty.Exit(2, "Could not create client for endpoint: %v, reason: %v", account.Endpoint, err) } - err = operations.DownloadCommand(client, account, workspaceId, robotId, zipfile, common.DebugFlag) + err = operations.DownloadCommand(client, account, workspaceId, robotId, zipfile, common.DebugFlag()) if err != nil { pretty.Exit(3, "Error: %v", err) } diff --git a/cmd/env.go b/cmd/env.go deleted file mode 100644 index 65d84d05..00000000 --- a/cmd/env.go +++ /dev/null @@ -1,17 +0,0 @@ -package cmd - -import ( - "github.com/spf13/cobra" -) - -var envCmd = &cobra.Command{ - Use: "env", - Aliases: []string{"environment", "e"}, - Short: "Group of commands related to `environment management`.", - Long: `This "env" command set is for managing virtual environments -used in task context locally.`, -} - -func init() { - rootCmd.AddCommand(envCmd) -} diff --git a/cmd/envNew.go b/cmd/envNew.go deleted file mode 100644 index 3eada719..00000000 --- a/cmd/envNew.go +++ /dev/null @@ -1,41 +0,0 @@ -package cmd - -import ( - "github.com/robocorp/rcc/common" - "github.com/robocorp/rcc/conda" - "github.com/robocorp/rcc/pretty" - - "github.com/spf13/cobra" -) - -var newEnvCmd = &cobra.Command{ - Use: "new ", - Short: "Creates a new managed virtual environment.", - Long: `The new command can be used to create a new managed virtual environment. -When given multiple conda.yaml files, they will be merged together and the -end result will be a composite environment.`, - Args: cobra.MinimumNArgs(1), - Run: func(cmd *cobra.Command, args []string) { - if common.DebugFlag { - defer common.Stopwatch("New environment creation lasted").Report() - } - ok := conda.MustConda() - if !ok { - pretty.Exit(2, "Could not get miniconda installed.") - } - label, err := conda.NewEnvironment(forceFlag, args...) - if err != nil { - pretty.Exit(1, "Environment creation failed: %v", err) - } else { - common.Log("Environment for %v as %v created.", args, label) - } - if common.Silent { - common.Stdout("%s\n", label) - } - }, -} - -func init() { - envCmd.AddCommand(newEnvCmd) - newEnvCmd.Flags().BoolVarP(&forceFlag, "force", "f", false, "Force conda cache update. (only for new environments)") -} diff --git a/cmd/events.go b/cmd/events.go new file mode 100644 index 00000000..b8a25c46 --- /dev/null +++ b/cmd/events.go @@ -0,0 +1,46 @@ +package cmd + +import ( + "encoding/json" + "fmt" + "os" + "text/tabwriter" + + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/journal" + "github.com/robocorp/rcc/pretty" + "github.com/spf13/cobra" +) + +func humaneEventListing(events []journal.Event) { + tabbed := tabwriter.NewWriter(os.Stderr, 2, 4, 2, ' ', 0) + tabbed.Write([]byte("When\tController\tEvent\tDetail\tComment\n")) + tabbed.Write([]byte("----\t----------\t-----\t------\t-------\n")) + for _, event := range events { + data := fmt.Sprintf("%d\t%s\t%s\t%s\t%s\n", event.When, event.Controller, event.Event, event.Detail, event.Comment) + tabbed.Write([]byte(data)) + } + tabbed.Flush() +} + +var eventsCmd = &cobra.Command{ + Use: "events", + Short: fmt.Sprintf("Show events from event journal (%s/event.log).", common.Product.HomeVariable()), + Long: fmt.Sprintf("Show events from event journal (%s/event.log).", common.Product.HomeVariable()), + Run: func(cmd *cobra.Command, args []string) { + events, err := journal.Events() + pretty.Guard(err == nil, 2, "Error while loading events: %v", err) + if jsonFlag { + output, err := json.MarshalIndent(events, "", " ") + pretty.Guard(err == nil, 3, "Error while converting events: %v", err) + fmt.Fprintln(os.Stdout, string(output)) + } else { + humaneEventListing(events) + } + }, +} + +func init() { + configureCmd.AddCommand(eventsCmd) + eventsCmd.Flags().BoolVarP(&jsonFlag, "json", "j", false, "Show effective settings as JSON stream.") +} diff --git a/cmd/feedback.go b/cmd/feedback.go index 2ca8f184..a907e50e 100644 --- a/cmd/feedback.go +++ b/cmd/feedback.go @@ -1,9 +1,26 @@ package cmd import ( + "fmt" + + "github.com/robocorp/rcc/cloud" + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/operations" + "github.com/robocorp/rcc/pretty" + "github.com/robocorp/rcc/xviper" "github.com/spf13/cobra" ) +var ( + issueRobot string + issueMetafile string + issueAttachments []string + + metricType string + metricName string + metricValue string +) + var feedbackCmd = &cobra.Command{ Use: "feedback", Aliases: []string{"f"}, @@ -12,6 +29,80 @@ var feedbackCmd = &cobra.Command{ Hidden: true, } +var issueCmd = &cobra.Command{ + Use: "issue", + Short: fmt.Sprintf("Send an issue to %s Control Room via rcc.", common.Product.Name()), + Long: fmt.Sprintf("Send an issue to %s Control Room via rcc.", common.Product.Name()), + Run: func(cmd *cobra.Command, args []string) { + if common.DebugFlag() { + defer common.Stopwatch("Feedback issue lasted").Report() + } + accountEmail := "unknown" + account := operations.AccountByName(AccountName()) + if account != nil && account.Details != nil { + email, ok := account.Details["email"].(string) + if ok { + accountEmail = email + } + } + err := operations.ReportIssue(accountEmail, issueRobot, issueMetafile, issueAttachments, dryFlag) + if err != nil { + pretty.Exit(1, "Error: %s", err) + } + pretty.Exit(0, "OK") + }, +} + +var metricCmd = &cobra.Command{ + Use: "metric", + Short: fmt.Sprintf("Send some metric to %s Control Room.", common.Product.Name()), + Long: fmt.Sprintf("Send some metric to %s Control Room.", common.Product.Name()), + Run: func(cmd *cobra.Command, args []string) { + if common.DebugFlag() { + defer common.Stopwatch("Feedback metric lasted").Report() + } + if !xviper.CanTrack() { + pretty.Exit(1, "Tracking is disabled. Quitting.") + } + cloud.BackgroundMetric(metricType, metricName, metricValue) + pretty.Exit(0, "OK") + }, +} + +var batchMetricCmd = &cobra.Command{ + Use: "batch ", + Short: fmt.Sprintf("Send batch metrics to %s Control Room. For applications only.", common.Product.Name()), + Long: fmt.Sprintf("Send batch metrics to %s Control Room. For applications only.", common.Product.Name()), + Args: cobra.ExactArgs(1), + Run: func(cmd *cobra.Command, args []string) { + if common.DebugFlag() { + defer common.Stopwatch("Feedback batch lasted").Report() + } + if xviper.CanTrack() { + cloud.BatchMetric(args[0]) + } else { + pretty.Warning("Tracking is disabled. Quitting.") + } + }, +} + func init() { rootCmd.AddCommand(feedbackCmd) + + feedbackCmd.AddCommand(issueCmd) + issueCmd.Flags().StringVarP(&issueMetafile, "report", "r", "", "Report file in JSON form containing actual issue report details.") + issueCmd.MarkFlagRequired("report") + issueCmd.Flags().StringArrayVarP(&issueAttachments, "attachments", "a", []string{}, "Files to attach to issue report.") + issueCmd.Flags().BoolVarP(&dryFlag, "dryrun", "d", false, "Don't send issue report, just show what would report be.") + issueCmd.Flags().StringVarP(&issueRobot, "robot", "", "", "Full path to 'robot.yaml' configuration file. [optional]") + + feedbackCmd.AddCommand(metricCmd) + metricCmd.Flags().StringVarP(&metricType, "type", "t", "", "Type for metric source to use.") + metricCmd.MarkFlagRequired("type") + metricCmd.Flags().StringVarP(&metricName, "name", "n", "", "Name for metric to report.") + metricCmd.MarkFlagRequired("name") + metricCmd.Flags().StringVarP(&metricValue, "value", "v", "", "Value for metric to report.") + metricCmd.MarkFlagRequired("value") + + feedbackCmd.AddCommand(batchMetricCmd) } diff --git a/cmd/finder.go b/cmd/finder.go index 967e8f80..ff951279 100644 --- a/cmd/finder.go +++ b/cmd/finder.go @@ -20,7 +20,7 @@ Example: rcc internal finder -d /starting/path/somewhere robot.yaml`, Args: cobra.ExactArgs(1), Run: func(cmd *cobra.Command, args []string) { - if common.DebugFlag { + if common.DebugFlag() { defer common.Stopwatch("Finder run lasted").Report() } found, err := pathlib.FindNamedPath(shellDirectory, args[0]) diff --git a/cmd/fix.go b/cmd/fix.go deleted file mode 100644 index 8c511877..00000000 --- a/cmd/fix.go +++ /dev/null @@ -1,32 +0,0 @@ -package cmd - -import ( - "github.com/robocorp/rcc/common" - "github.com/robocorp/rcc/operations" - "github.com/robocorp/rcc/pretty" - - "github.com/spf13/cobra" -) - -var fixCmd = &cobra.Command{ - Use: "fix", - Short: "Automatically fix known issues inside robots.", - Long: `Automatically fix known issues inside robots. Current fixes are: -- make files in PATH folder executable -- convert .sh newlines to unix form`, - Run: func(cmd *cobra.Command, args []string) { - if common.DebugFlag { - defer common.Stopwatch("Fix run lasted").Report() - } - err := operations.FixRobot(robotFile) - if err != nil { - pretty.Exit(1, "Error: %v", err) - } - pretty.Ok() - }, -} - -func init() { - robotCmd.AddCommand(fixCmd) - fixCmd.Flags().StringVarP(&robotFile, "robot", "r", "robot.yaml", "Full path to 'robot.yaml' configuration file.") -} diff --git a/cmd/holotree.go b/cmd/holotree.go new file mode 100644 index 00000000..331f3d32 --- /dev/null +++ b/cmd/holotree.go @@ -0,0 +1,23 @@ +package cmd + +import ( + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/settings" + "github.com/spf13/cobra" +) + +var holotreeCmd = &cobra.Command{ + Use: "holotree", + Aliases: []string{"ht"}, + Short: "Group of holotree commands.", + Long: "Group of holotree commands.", + PersistentPreRun: func(cmd *cobra.Command, args []string) { + settings.CriticalEnvironmentSettingsCheck() + }, +} + +func init() { + rootCmd.AddCommand(holotreeCmd) + + holotreeCmd.PersistentFlags().BoolVarP(&common.ExternallyManaged, "externally-managed", "", false, "mark created Python environments as EXTERNALLY-MANAGED (PEP 668)") +} diff --git a/cmd/holotreeBlueprints.go b/cmd/holotreeBlueprints.go new file mode 100644 index 00000000..a3525fb0 --- /dev/null +++ b/cmd/holotreeBlueprints.go @@ -0,0 +1,55 @@ +package cmd + +import ( + "fmt" + + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/htfs" + "github.com/robocorp/rcc/operations" + "github.com/robocorp/rcc/pretty" + "github.com/spf13/cobra" +) + +func holotreeExpandBlueprint(userFiles []string, packfile string) map[string]interface{} { + result := make(map[string]interface{}) + + _, holotreeBlueprint, err := htfs.ComposeFinalBlueprint(userFiles, packfile) + pretty.Guard(err == nil, 5, "%s", err) + + common.Debug("FINAL blueprint:\n%s", string(holotreeBlueprint)) + + tree, err := htfs.New() + pretty.Guard(err == nil, 6, "%s", err) + + result["hash"] = common.BlueprintHash(holotreeBlueprint) + result["exist"] = tree.HasBlueprint(holotreeBlueprint) + + return result +} + +var holotreeBlueprintCmd = &cobra.Command{ + Use: "blueprint conda.yaml+", + Short: "Verify that resulting blueprint is in hololibrary.", + Long: "Verify that resulting blueprint is in hololibrary.", + Aliases: []string{"bp"}, + Run: func(cmd *cobra.Command, args []string) { + if common.DebugFlag() { + defer common.Stopwatch("Holotree blueprints command lasted").Report() + } + + status := holotreeExpandBlueprint(args, robotFile) + if holotreeJson { + out, err := operations.NiceJsonOutput(status) + pretty.Guard(err == nil, 6, "%s", err) + fmt.Println(out) + } else { + common.Log("Blueprint %q is available: %v", status["hash"], status["exist"]) + } + }, +} + +func init() { + holotreeCmd.AddCommand(holotreeBlueprintCmd) + holotreeBlueprintCmd.Flags().StringVarP(&robotFile, "robot", "r", "robot.yaml", "Full path to 'robot.yaml' configuration file. ") + holotreeBlueprintCmd.Flags().BoolVarP(&holotreeJson, "json", "j", false, "Show environment as JSON.") +} diff --git a/cmd/holotreeBootstrap.go b/cmd/holotreeBootstrap.go new file mode 100644 index 00000000..900985f2 --- /dev/null +++ b/cmd/holotreeBootstrap.go @@ -0,0 +1,69 @@ +package cmd + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/htfs" + "github.com/robocorp/rcc/operations" + "github.com/robocorp/rcc/pathlib" + "github.com/robocorp/rcc/pretty" + "github.com/robocorp/rcc/robot" + "github.com/spf13/cobra" +) + +var ( + holotreeQuick bool +) + +func updateEnvironments(robots []string) { + tree, err := htfs.New() + pretty.Guard(err == nil, 2, "Holotree creation error: %v", err) + for at, template := range robots { + workarea := filepath.Join(pathlib.TempDir(), fmt.Sprintf("workarea%x%x", common.When, at)) + defer os.RemoveAll(workarea) + common.Debug("Using temporary workarea: %v", workarea) + err = operations.InitializeWorkarea(workarea, template, false, forceFlag) + pretty.Guard(err == nil, 2, "Could not create robot %q, reason: %v", template, err) + targetRobot := robot.DetectConfigurationName(workarea) + _, blueprint, err := htfs.ComposeFinalBlueprint([]string{}, targetRobot) + if tree.HasBlueprint(blueprint) { + continue + } + config, err := robot.LoadRobotYaml(targetRobot, false) + pretty.Guard(err == nil, 2, "Could not load robot config %q, reason: %w", targetRobot, err) + if !config.UsesConda() { + continue + } + _, _, err = htfs.NewEnvironment(config.CondaConfigFile(), "", false, false, operations.PullCatalog) + pretty.Guard(err == nil, 2, "Holotree recording error: %v", err) + } +} + +var holotreeBootstrapCmd = &cobra.Command{ + Use: "bootstrap", + Aliases: []string{"boot"}, + Short: "Bootstrap holotree from set of templates.", + Long: "Bootstrap holotree from set of templates.", + Args: cobra.NoArgs, + Run: func(cmd *cobra.Command, args []string) { + if common.DebugFlag() { + defer common.Stopwatch("Holotree bootstrap lasted").Report() + } + + robots := operations.ListTemplates(false) + + if !holotreeQuick { + updateEnvironments(robots) + } + + pretty.Ok() + }, +} + +func init() { + holotreeCmd.AddCommand(holotreeBootstrapCmd) + holotreeBootstrapCmd.Flags().BoolVar(&holotreeQuick, "quick", false, "Do not create environments, just download templates.") +} diff --git a/cmd/holotreeCatalogs.go b/cmd/holotreeCatalogs.go new file mode 100644 index 00000000..b41cf1f9 --- /dev/null +++ b/cmd/holotreeCatalogs.go @@ -0,0 +1,184 @@ +package cmd + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "text/tabwriter" + + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/htfs" + "github.com/robocorp/rcc/pathlib" + "github.com/robocorp/rcc/pretty" + "github.com/robocorp/rcc/set" + "github.com/spf13/cobra" +) + +var ( + showIdentityYaml bool + topSizes int +) + +const mega = 1024 * 1024 + +func megas(bytes uint64) uint64 { + return bytes / mega +} + +func catalogUsedStats() map[string]int { + result := make(map[string]int) + handle, err := os.Open(common.HololibUsageLocation()) + if err != nil { + return result + } + defer handle.Close() + entries, err := handle.Readdir(-1) + if err != nil { + return result + } + for _, entry := range entries { + name := filepath.Base(entry.Name()) + tail := filepath.Ext(name) + size := len(name) - len(tail) + base := name[:size] + days := common.DayCountSince(entry.ModTime()) + previous, ok := result[base] + if !ok || days < previous { + result[base] = days + } + } + return result +} + +func identityContent(catalog *htfs.Root) string { + blob, err := catalog.Show("identity.yaml") + if err != nil { + return err.Error() + } + return string(blob) +} + +func identityContentLines(catalog *htfs.Root) []string { + content := identityContent(catalog) + result := strings.SplitAfter(content, "\n") + for at, value := range result { + result[at] = strings.Replace(strings.TrimRight(value, "\r\n\t "), "\t", " ", -1) + } + return result +} + +func jsonCatalogDetails(roots []*htfs.Root, topN int) { + used := catalogUsedStats() + holder := make(map[string]map[string]interface{}) + for _, catalog := range roots { + lastUse, ok := used[catalog.Blueprint] + if !ok { + catalog.Touch() + lastUse = -1 + } + stats, err := catalog.Stats() + pretty.Guard(err == nil, 1, "Could not get stats for %s, reason: %s", catalog.Blueprint, err) + data := make(map[string]interface{}) + data["blueprint"] = catalog.Blueprint + data["holotree"] = catalog.HolotreeBase() + identity := filepath.Join(common.HololibLibraryLocation(), stats.Identity) + data["identity.yaml"] = identity + if showIdentityYaml { + data["identity-content"] = identityContent(catalog) + } + if topN > 0 { + data[fmt.Sprintf("top%d", topN)] = catalog.Top(topN) + } + data["platform"] = catalog.Platform + data["directories"] = stats.Directories + data["files"] = stats.Files + data["bytes"] = stats.Bytes + data["relocations"] = stats.Relocations + holder[catalog.Blueprint] = data + age, _ := pathlib.DaysSinceModified(catalog.Source()) + data["age_in_days"] = age + data["days_since_last_use"] = lastUse + } + nice, err := json.MarshalIndent(holder, "", " ") + pretty.Guard(err == nil, 2, "%s", err) + common.Stdout("%s\n", nice) +} + +func percent(value, base float64) float64 { + if base == 0.0 { + return 0.0 + } + return 100.0 * value / base +} + +func dumpTopN(stats map[string]int64, total float64, tabbed *tabwriter.Writer) { + sizes := set.Values(stats) + sort.Slice(sizes, func(left, right int) bool { + return sizes[left] > sizes[right] + }) + for _, focus := range sizes { + share := percent(float64(focus), total) + value, suffix := pathlib.HumaneSizer(focus) + for filename, size := range stats { + if focus == size { + tabbed.Write([]byte(fmt.Sprintf("\t\t\t%5.1f%%\t%6.1f%s\t%s\n", share, value, suffix, filename))) + } + } + } +} + +func listCatalogDetails(roots []*htfs.Root, topN int) { + used := catalogUsedStats() + tabbed := tabwriter.NewWriter(os.Stderr, 2, 4, 2, ' ', 0) + tabbed.Write([]byte("Blueprint\tPlatform\tDirs \tFiles \tSize \tRelocate\tidentity.yaml (gzipped blob inside hololib)\tHolotree path\tAge (days)\tIdle (days)\n")) + tabbed.Write([]byte("---------\t--------\t------\t-------\t-------\t--------\t-------------------------------------------\t-------------\t----------\t-----------\n")) + for _, catalog := range roots { + lastUse, ok := used[catalog.Blueprint] + if !ok { + catalog.Touch() + lastUse = -1 + } + stats, err := catalog.Stats() + pretty.Guard(err == nil, 1, "Could not get stats for %s, reason: %s", catalog.Blueprint, err) + days, _ := pathlib.DaysSinceModified(catalog.Source()) + data := fmt.Sprintf("%s\t%s\t% 6d\t% 7d\t% 6dM\t% 8d\t%s\t%s\t%10d\t%11d\n", catalog.Blueprint, catalog.Platform, stats.Directories, stats.Files, megas(stats.Bytes), stats.Relocations, stats.Identity, catalog.HolotreeBase(), days, lastUse) + tabbed.Write([]byte(data)) + if showIdentityYaml { + for _, line := range identityContentLines(catalog) { + tabbed.Write([]byte(fmt.Sprintf("\t\t\t\t\t\t%s\n", line))) + } + } + if topN > 0 { + dumpTopN(catalog.Top(topN), float64(stats.Bytes), tabbed) + } + } + tabbed.Flush() +} + +var holotreeCatalogsCmd = &cobra.Command{ + Use: "catalogs", + Short: "List native and imported holotree catalogs.", + Long: "List native and imported holotree catalogs.", + Run: func(cmd *cobra.Command, args []string) { + if common.DebugFlag() { + defer common.Stopwatch("Holotree catalogs command lasted").Report() + } + _, roots := htfs.LoadCatalogs() + if jsonFlag { + jsonCatalogDetails(roots, topSizes) + } else { + listCatalogDetails(roots, topSizes) + } + pretty.Ok() + }, +} + +func init() { + holotreeCmd.AddCommand(holotreeCatalogsCmd) + holotreeCatalogsCmd.Flags().BoolVarP(&jsonFlag, "json", "j", false, "Output in JSON format") + holotreeCatalogsCmd.Flags().BoolVarP(&showIdentityYaml, "identity", "i", false, "Show identity.yaml in catalog context.") + holotreeCatalogsCmd.Flags().IntVarP(&topSizes, "top", "t", 0, "Show top N sized files from catalog") +} diff --git a/cmd/holotreeCheck.go b/cmd/holotreeCheck.go new file mode 100644 index 00000000..285a6cd2 --- /dev/null +++ b/cmd/holotreeCheck.go @@ -0,0 +1,101 @@ +package cmd + +import ( + "fmt" + "path/filepath" + + "github.com/robocorp/rcc/anywork" + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/fail" + "github.com/robocorp/rcc/htfs" + "github.com/robocorp/rcc/pathlib" + "github.com/robocorp/rcc/pretty" + "github.com/spf13/cobra" +) + +var ( + checkRetries int +) + +func checkHolotreeIntegrity() (err error) { + defer fail.Around(&err) + + common.Timeline("holotree integrity check start") + defer common.Timeline("holotree integrity check done") + fs, err := htfs.NewRoot(common.HololibLibraryLocation()) + fail.On(err != nil, "%s", err) + common.Timeline("holotree integrity lift") + err = fs.Lift() + fail.On(err != nil, "%s", err) + common.Timeline("holotree integrity hasher") + known, needed := htfs.LoadHololibHashes() + err = fs.AllFiles(htfs.CheckHasher(known)) + fail.On(err != nil, "%s", err) + collector := make(map[string]string) + common.Timeline("holotree integrity collector") + err = fs.Treetop(htfs.IntegrityCheck(collector, needed)) + common.Timeline("holotree integrity report") + fail.On(err != nil, "%s", err) + purge := make(map[string]bool) + for k, _ := range collector { + found, ok := known[filepath.Base(k)] + if !ok { + continue + } + for catalog, _ := range found { + purge[catalog] = true + } + } + for _, v := range needed { + for catalog, _ := range v { + purge[catalog] = true + } + } + redo := false + for k, _ := range purge { + fmt.Println("Purge catalog:", k) + redo = true + anywork.Backlog(htfs.RemoveFile(k)) + } + err = anywork.Sync() + fail.On(err != nil, "%s", err) + fail.On(redo, "Some catalogs were purged. Run this check command again, please!") + fail.On(len(collector) > 0, "Size: %d", len(collector)) + err = pathlib.RemoveEmptyDirectores(common.HololibLibraryLocation()) + fail.On(err != nil, "%s", err) + return nil +} + +func checkLoop(retryCount int) { + var err error +loop: + for retryCount > 0 { + retryCount-- + err = checkHolotreeIntegrity() + if err == nil { + break loop + } + common.Timeline("!!! holotree integrity retry needed [remaining: %d]", retryCount) + } + pretty.Guard(err == nil, 1, "%s", err) +} + +var holotreeCheckCmd = &cobra.Command{ + Use: "check", + Short: "Check holotree library integrity.", + Long: "Check holotree library integrity.", + Aliases: []string{"chk"}, + Run: func(cmd *cobra.Command, args []string) { + repeat := 1 + if checkRetries > 0 { + repeat += checkRetries + } + checkLoop(repeat) + pretty.Ok() + }, +} + +func init() { + holotreeCheckCmd.Flags().IntVarP(&checkRetries, "retries", "r", 1, "How many retries to do in case of failures.") + holotreeCmd.AddCommand(holotreeCheckCmd) +} diff --git a/cmd/holotreeDelete.go b/cmd/holotreeDelete.go new file mode 100644 index 00000000..c04a15c6 --- /dev/null +++ b/cmd/holotreeDelete.go @@ -0,0 +1,54 @@ +package cmd + +import ( + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/htfs" + "github.com/robocorp/rcc/pretty" + + "github.com/spf13/cobra" +) + +var ( + deleteSpace string +) + +func deleteByPartialIdentity(partials []string) { + _, roots := htfs.LoadCatalogs() + var note string + if dryFlag { + note = "[dry run] " + } + for _, label := range roots.FindEnvironments(partials) { + common.Log("%sRemoving %v", note, label) + if dryFlag { + continue + } + err := roots.RemoveHolotreeSpace(label) + pretty.Guard(err == nil, 1, "Error: %v", err) + } +} + +var holotreeDeleteCmd = &cobra.Command{ + Use: "delete *", + Short: "Delete one or more holotree controller spaces.", + Long: "Delete one or more holotree controller spaces.", + Aliases: []string{"del"}, + Run: func(cmd *cobra.Command, args []string) { + partials := make([]string, 0, len(args)+1) + if len(args) > 0 { + partials = append(partials, args...) + } + if len(deleteSpace) > 0 { + partials = append(partials, htfs.ControllerSpaceName([]byte(common.ControllerIdentity()), []byte(deleteSpace))) + } + pretty.Guard(len(partials) > 0, 1, "Must provide either --space flag, or partial environment identity!") + deleteByPartialIdentity(partials) + pretty.Ok() + }, +} + +func init() { + holotreeCmd.AddCommand(holotreeDeleteCmd) + holotreeDeleteCmd.Flags().BoolVarP(&dryFlag, "dryrun", "d", false, "Don't delete environments, just show what would happen.") + holotreeDeleteCmd.Flags().StringVarP(&deleteSpace, "space", "s", "", "Client specific name to identify environment to delete.") +} diff --git a/cmd/holotreeExport.go b/cmd/holotreeExport.go new file mode 100644 index 00000000..adbdb8ce --- /dev/null +++ b/cmd/holotreeExport.go @@ -0,0 +1,92 @@ +package cmd + +import ( + "encoding/json" + "sort" + "strings" + + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/htfs" + "github.com/robocorp/rcc/pretty" + "github.com/spf13/cobra" +) + +var ( + holozip string + exportRobot string +) + +func holotreeExport(catalogs, known []string, archive string) { + common.Debug("Ignoring content from catalogs:") + for _, catalog := range known { + common.Debug("- %s", catalog) + } + + common.Debug("Exporting catalogs:") + for _, catalog := range catalogs { + common.Debug("- %s", catalog) + } + + tree, err := htfs.New() + pretty.Guard(err == nil, 2, "%s", err) + + err = tree.Export(catalogs, known, archive) + pretty.Guard(err == nil, 3, "%s", err) +} + +func listCatalogs(jsonForm bool) { + if jsonForm { + nice, err := json.MarshalIndent(htfs.CatalogNames(), "", " ") + pretty.Guard(err == nil, 2, "%s", err) + common.Stdout("%s\n", nice) + } else { + common.Log("Selectable catalogs (you can use substrings):") + for _, catalog := range htfs.CatalogNames() { + common.Log("- %s", catalog) + } + } +} + +func selectCatalogs(filters []string) []string { + result := make([]string, 0, len(filters)) + for _, catalog := range htfs.CatalogNames() { + for _, filter := range filters { + if strings.Contains(catalog, filter) { + result = append(result, catalog) + break + } + } + } + sort.Strings(result) + return result +} + +var holotreeExportCmd = &cobra.Command{ + Use: "export catalog+", + Short: "Export existing holotree catalog and library parts.", + Long: "Export existing holotree catalog and library parts.", + Run: func(cmd *cobra.Command, args []string) { + if common.DebugFlag() { + defer common.Stopwatch("Holotree export command lasted").Report() + } + if len(exportRobot) > 0 { + _, holotreeBlueprint, err := htfs.ComposeFinalBlueprint(nil, exportRobot) + pretty.Guard(err == nil, 1, "Blueprint calculation failed: %v", err) + hash := common.BlueprintHash(holotreeBlueprint) + args = append(args, htfs.CatalogName(hash)) + } + if len(args) == 0 { + listCatalogs(jsonFlag) + } else { + holotreeExport(selectCatalogs(args), nil, holozip) + } + pretty.Ok() + }, +} + +func init() { + holotreeCmd.AddCommand(holotreeExportCmd) + holotreeExportCmd.Flags().StringVarP(&holozip, "zipfile", "z", "hololib.zip", "Name of zipfile to export.") + holotreeExportCmd.Flags().BoolVarP(&jsonFlag, "json", "j", false, "Output in JSON format") + holotreeExportCmd.Flags().StringVarP(&exportRobot, "robot", "r", "", "Full path to 'robot.yaml' configuration file to export as catalog. ") +} diff --git a/cmd/holotreeHash.go b/cmd/holotreeHash.go new file mode 100644 index 00000000..4884846f --- /dev/null +++ b/cmd/holotreeHash.go @@ -0,0 +1,32 @@ +package cmd + +import ( + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/htfs" + "github.com/robocorp/rcc/pretty" + + "github.com/spf13/cobra" +) + +var holotreeHashCmd = &cobra.Command{ + Use: "hash ", + Short: "Calculates a blueprint hash for managed holotree virtual environment from conda.yaml files.", + Long: "Calculates a blueprint hash for managed holotree virtual environment from conda.yaml files.", + Args: cobra.MinimumNArgs(1), + Run: func(cmd *cobra.Command, args []string) { + if common.DebugFlag() { + defer common.Stopwatch("Conda YAML hash calculation lasted").Report() + } + _, holotreeBlueprint, err := htfs.ComposeFinalBlueprint(args, "") + pretty.Guard(err == nil, 1, "Blueprint calculation failed: %v", err) + hash := common.BlueprintHash(holotreeBlueprint) + common.Log("Blueprint hash for %v is %v.", args, hash) + if common.Silent() { + common.Stdout("%s\n", hash) + } + }, +} + +func init() { + holotreeCmd.AddCommand(holotreeHashCmd) +} diff --git a/cmd/holotreeImport.go b/cmd/holotreeImport.go new file mode 100644 index 00000000..3379b937 --- /dev/null +++ b/cmd/holotreeImport.go @@ -0,0 +1,80 @@ +package cmd + +import ( + "fmt" + "net/url" + "os" + "path/filepath" + + "github.com/robocorp/rcc/cloud" + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/operations" + "github.com/robocorp/rcc/pathlib" + "github.com/robocorp/rcc/pretty" + "github.com/spf13/cobra" +) + +func isUrl(name string) bool { + link, err := url.Parse(name) + if err != nil { + return false + } + return link.IsAbs() && (link.Scheme == "http" || link.Scheme == "https") +} + +func temporaryDownload(at int, link string) (string, error) { + common.Timeline("Download %v", link) + zipfile := filepath.Join(pathlib.TempDir(), fmt.Sprintf("hololib%x%x.zip", common.When, at)) + err := cloud.Download(link, zipfile) + if err != nil { + return "", err + } + return zipfile, nil +} + +func reportAllErrors(filename string, errors []error) error { + if errors == nil || len(errors) == 0 { + return nil + } + if len(errors) == 1 { + return errors[0] + } + common.Log("Errors from zip %q:", filename) + for at, err := range errors { + common.Log("- %d: %v", at+1, err) + } + return errors[0] +} + +var holotreeImportCmd = &cobra.Command{ + Use: "import hololib.zip+", + Short: "Import one or more hololib.zip files into local hololib.", + Long: "Import one or more hololib.zip files into local hololib.", + Args: cobra.MinimumNArgs(1), + Run: func(cmd *cobra.Command, args []string) { + var err error + + if common.DebugFlag() { + defer common.Stopwatch("Holotree import command lasted").Report() + } + for at, filename := range args { + if isUrl(filename) { + filename, err = temporaryDownload(at, filename) + pretty.Guard(err == nil, 2, "Could not download %q, reason: %v", filename, err) + defer os.Remove(filename) + } + if common.StrictFlag { + errors := operations.VerifyZip(filename, operations.HololibZipShape) + err = reportAllErrors(filename, errors) + pretty.Guard(err == nil, 3, "Could not verify %q, first reason: %v", filename, err) + } + err = operations.ProtectedImport(filename) + pretty.Guard(err == nil, 1, "Could not import %q, reason: %v", filename, err) + } + pretty.Ok() + }, +} + +func init() { + holotreeCmd.AddCommand(holotreeImportCmd) +} diff --git a/cmd/holotreeInit.go b/cmd/holotreeInit.go new file mode 100644 index 00000000..47111aff --- /dev/null +++ b/cmd/holotreeInit.go @@ -0,0 +1,56 @@ +package cmd + +import ( + "os" + + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/pathlib" + "github.com/robocorp/rcc/pretty" + + "github.com/spf13/cobra" +) + +var ( + revokeInit bool +) + +func disableHolotreeSharing() { + pretty.Guard(common.SharedHolotree, 5, "Not using shared holotree. Cannot disable either.") + err := os.Remove(common.HoloInitUserFile()) + pretty.Guard(err == nil, 6, "Could not remove shared user file at %q, reason: %v", common.HoloInitUserFile(), err) +} + +func enableHolotreeSharing() { + pathlib.ForceShared() + _, err := pathlib.ForceSharedDir(common.HoloInitLocation()) + pretty.Guard(err == nil, 1, "Could not enable shared location at %q, reason: %v", common.HoloInitLocation(), err) + err = os.WriteFile(common.HoloInitCommonFile(), []byte("OK!"), 0o666) + pretty.Guard(err == nil, 2, "Could not write shared common file at %q, reason: %v", common.HoloInitCommonFile(), err) + err = os.WriteFile(common.HoloInitUserFile(), []byte("OK!"), 0o640) + pretty.Guard(err == nil, 3, "Could not write shared user file at %q, reason: %v", common.HoloInitUserFile(), err) + _, err = pathlib.MakeSharedFile(common.HoloInitCommonFile()) + pretty.Guard(err == nil, 4, "Could not make shared common file actually shared at %q, reason: %v", common.HoloInitCommonFile(), err) +} + +var holotreeInitCmd = &cobra.Command{ + Use: "init", + Short: "Initialize shared holotree location.", + Long: "Initialize shared holotree location.", + Run: func(cmd *cobra.Command, args []string) { + if common.DebugFlag() { + defer common.Stopwatch("Initialize shared holotree location lasted").Report() + } + pretty.Warning("Running this command might need 'rcc holotree shared --enable' first. Still, trying ...") + if revokeInit { + disableHolotreeSharing() + } else { + enableHolotreeSharing() + } + pretty.Ok() + }, +} + +func init() { + holotreeInitCmd.Flags().BoolVarP(&revokeInit, "revoke", "r", false, "Revoke shared holotree usage. Go back to private holotree usage.") + holotreeCmd.AddCommand(holotreeInitCmd) +} diff --git a/cmd/holotreeList.go b/cmd/holotreeList.go new file mode 100644 index 00000000..94aa1ef2 --- /dev/null +++ b/cmd/holotreeList.go @@ -0,0 +1,91 @@ +package cmd + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "text/tabwriter" + "time" + + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/htfs" + "github.com/robocorp/rcc/pretty" + "github.com/spf13/cobra" +) + +func whatUsage(space string) (string, string, int) { + usefile := fmt.Sprintf("%s.use", space) + stat, err := os.Stat(usefile) + if err != nil { + return "N/A", "N/A", 0 + } + times := fmt.Sprintf("%d times", stat.Size()) + delta := time.Now().Sub(stat.ModTime()).Hours() / 24.0 + when := fmt.Sprintf("%1.0f days ago", delta) + return when, times, int(delta) +} + +func humaneHolotreeSpaceListing() { + tabbed := tabwriter.NewWriter(os.Stderr, 2, 4, 2, ' ', 0) + tabbed.Write([]byte("Identity\tController\tSpace\tBlueprint\tFull path\tLast used\tUse count\n")) + tabbed.Write([]byte("--------\t----------\t-----\t---------\t---------\t---------\t---------\n")) + _, roots := htfs.LoadCatalogs() + for _, space := range roots.Spaces() { + when, times, _ := whatUsage(space.Path) + data := fmt.Sprintf("%s\t%s\t%s\t%s\t%s\t%s\t%s\n", space.Identity, space.Controller, space.Space, space.Blueprint, space.Path, when, times) + tabbed.Write([]byte(data)) + } + tabbed.Flush() +} + +func jsonicHolotreeSpaceListing() { + details := make(map[string]map[string]any) + _, roots := htfs.LoadCatalogs() + for _, space := range roots.Spaces() { + hold, ok := details[space.Identity] + if !ok { + hold = make(map[string]any) + details[space.Identity] = hold + hold["id"] = space.Identity + hold["controller"] = space.Controller + hold["space"] = space.Space + hold["blueprint"] = space.Blueprint + hold["path"] = space.Path + hold["meta"] = space.Path + ".meta" + hold["spec"] = filepath.Join(space.Path, "identity.yaml") + hold["plan"] = filepath.Join(space.Path, "rcc_plan.log") + when, times, idle := whatUsage(space.Path) + hold["last-used"] = when + hold["idle-days"] = idle + hold["use-count"] = times + } + } + body, err := json.MarshalIndent(details, "", " ") + pretty.Guard(err == nil, 1, "Could not create json, reason: %w", err) + fmt.Println(string(body)) +} + +var holotreeListCmd = &cobra.Command{ + Use: "list", + Aliases: []string{"ls"}, + Short: "List holotree spaces.", + Long: "List holotree spaces.", + Run: func(cmd *cobra.Command, args []string) { + if common.DebugFlag() { + defer common.Stopwatch("Holotree list lasted").Report() + } + + if jsonFlag { + jsonicHolotreeSpaceListing() + } else { + humaneHolotreeSpaceListing() + } + + }, +} + +func init() { + holotreeCmd.AddCommand(holotreeListCmd) + holotreeListCmd.Flags().BoolVarP(&jsonFlag, "json", "j", false, "Output in JSON format") +} diff --git a/cmd/holotreePlan.go b/cmd/holotreePlan.go new file mode 100644 index 00000000..81cb3dfd --- /dev/null +++ b/cmd/holotreePlan.go @@ -0,0 +1,42 @@ +package cmd + +import ( + "io" + "os" + + "github.com/robocorp/rcc/conda" + "github.com/robocorp/rcc/htfs" + "github.com/robocorp/rcc/pretty" + + "github.com/spf13/cobra" +) + +var holotreePlanCmd = &cobra.Command{ + Use: "plan ", + Short: "Show installation plans for given holotree spaces (or substrings)", + Long: "Show installation plans for given holotree spaces (or substrings)", + Args: cobra.MinimumNArgs(1), + + Run: func(cmd *cobra.Command, args []string) { + found := false + _, roots := htfs.LoadCatalogs() + for _, label := range roots.FindEnvironments(args) { + planfile, ok := roots.InstallationPlan(label) + pretty.Guard(ok, 1, "Could not find plan for: %v", label) + source, err := os.Open(planfile) + pretty.Guard(err == nil, 2, "Could not read plan %q, reason: %v", planfile, err) + defer source.Close() + analyzer := conda.NewPlanAnalyzer(false) + defer analyzer.Close() + sink := io.MultiWriter(os.Stdout, analyzer) + io.Copy(sink, source) + found = true + } + pretty.Guard(found, 3, "Nothing matched given plans!") + pretty.Ok() + }, +} + +func init() { + holotreeCmd.AddCommand(holotreePlanCmd) +} diff --git a/cmd/holotreePrebuild.go b/cmd/holotreePrebuild.go new file mode 100644 index 00000000..f15f04b5 --- /dev/null +++ b/cmd/holotreePrebuild.go @@ -0,0 +1,151 @@ +package cmd + +import ( + "net/url" + "os" + "path/filepath" + "strings" + + "github.com/robocorp/rcc/cloud" + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/conda" + "github.com/robocorp/rcc/htfs" + "github.com/robocorp/rcc/operations" + "github.com/robocorp/rcc/pathlib" + "github.com/robocorp/rcc/pretty" + "github.com/robocorp/rcc/set" + "github.com/spf13/cobra" +) + +var ( + metafileFlag bool + forceBuild bool + exportFile string +) + +func conditionalExpand(filename string) string { + if !pathlib.IsFile(filename) { + return filename + } + fullpath, err := filepath.Abs(filename) + if err != nil { + return filename + } + return fullpath +} + +func resolveMetafileURL(link string) ([]string, error) { + origin, err := url.Parse(link) + refok := err == nil + raw, err := cloud.ReadFile(link) + if err != nil { + return nil, err + } + result := []string{} + for _, line := range strings.SplitAfter(string(raw), "\n") { + flat := strings.TrimSpace(line) + if strings.HasPrefix(flat, "#") || len(flat) == 0 { + continue + } + here, err := url.Parse(flat) + if refok && err == nil { + relative := origin.ResolveReference(here) + result = append(result, relative.String()) + } else { + result = append(result, flat) + } + } + return result, nil +} + +func resolveMetafile(link string) ([]string, error) { + if !pathlib.IsFile(link) { + return resolveMetafileURL(link) + } + fullpath, err := filepath.Abs(link) + if err != nil { + return nil, err + } + basedir := filepath.Dir(fullpath) + raw, err := os.ReadFile(fullpath) + if err != nil { + return nil, err + } + result := []string{} + for _, line := range strings.SplitAfter(string(raw), "\n") { + flat := strings.TrimSpace(line) + if strings.HasPrefix(flat, "#") || len(flat) == 0 { + continue + } + result = append(result, filepath.Join(basedir, flat)) + } + return result, nil +} + +func metafileExpansion(links []string, expand bool) []string { + if !expand { + return links + } + result := []string{} + for _, metalink := range links { + links, err := resolveMetafile(conditionalExpand(metalink)) + if err != nil { + pretty.Warning("Failed to resolve %q metafile, reason: %v", metalink, err) + continue + } + result = append(result, links...) + } + return result +} + +var holotreePrebuildCmd = &cobra.Command{ + Use: "prebuild", + Short: "Prebuild hololib from given set of environment descriptors.", + Long: "Prebuild hololib from given set of environment descriptors. Requires shared holotree to be enabled and active.", + Args: cobra.MinimumNArgs(1), + Run: func(cmd *cobra.Command, args []string) { + if common.DebugFlag() { + defer common.Stopwatch("Holotree prebuild lasted").Report() + } + + pretty.Guard(common.SharedHolotree, 1, "Shared holotree must be enabled and in use for prebuild environments to work correctly.") + + configurations := metafileExpansion(args, metafileFlag) + total, failed := len(configurations), 0 + success := make([]string, 0, total) + exporting := len(exportFile) > 0 + for at, configfile := range configurations { + environment, err := conda.ReadPackageCondaYaml(configfile) + if err != nil { + pretty.Warning("%d/%d: Failed to load %q, reason: %v (ignored)", at+1, total, configfile, err) + continue + } + pretty.Note("%d/%d: Now building config %q", at+1, total, configfile) + _, _, err = htfs.NewEnvironment(configfile, "", false, forceBuild, operations.PullCatalog) + if err != nil { + failed += 1 + pretty.Warning("%d/%d: Holotree recording error: %v", at+1, total, err) + } else { + for _, hash := range environment.FingerprintLayers() { + key := htfs.CatalogName(hash) + if exporting && !set.Member(success, key) { + success = append(success, key) + pretty.Note("Added catalog %q to be exported.", key) + } + } + } + } + if exporting && len(success) > 0 { + holotreeExport(selectCatalogs(success), nil, exportFile) + } + pretty.Guard(failed == 0, 2, "%d out of %d environment builds failed! See output above for details.", failed, total) + pretty.Ok() + }, +} + +func init() { + holotreeCmd.AddCommand(holotreePrebuildCmd) + holotreePrebuildCmd.Flags().BoolVarP(&metafileFlag, "metafile", "m", false, "Input arguments are actually files containing links/filenames of environment descriptors.") + holotreePrebuildCmd.Flags().BoolVarP(&forceBuild, "force", "f", false, "Force environment builds, even when blueprint is already present.") + holotreePrebuildCmd.Flags().StringVarP(&exportFile, "export", "e", "", "Optional filename to export new, successfully build catalogs.") +} diff --git a/cmd/holotreePull.go b/cmd/holotreePull.go new file mode 100644 index 00000000..a4688d5d --- /dev/null +++ b/cmd/holotreePull.go @@ -0,0 +1,50 @@ +package cmd + +import ( + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/htfs" + "github.com/robocorp/rcc/operations" + "github.com/robocorp/rcc/pretty" + "github.com/spf13/cobra" +) + +var ( + remoteOriginOption string + pullRobot string + forcePull bool +) + +var holotreePullCmd = &cobra.Command{ + Use: "pull", + Short: "Try to pull existing holotree catalog from remote source.", + Long: "Try to pull existing holotree catalog from remote source.", + Run: func(cmd *cobra.Command, args []string) { + if common.DebugFlag() { + defer common.Stopwatch("Holotree pull command lasted").Report() + } + _, holotreeBlueprint, err := htfs.ComposeFinalBlueprint(nil, pullRobot) + pretty.Guard(err == nil, 1, "Blueprint calculation failed: %v", err) + hash := common.BlueprintHash(holotreeBlueprint) + tree, err := htfs.New() + pretty.Guard(err == nil, 2, "%s", err) + + present := tree.HasBlueprint(holotreeBlueprint) + if !present || forcePull { + catalog := htfs.CatalogName(hash) + err = operations.PullCatalog(remoteOriginOption, catalog, true) + pretty.Guard(err == nil, 3, "%s", err) + } + pretty.Ok() + }, +} + +func init() { + origin := common.RccRemoteOrigin() + holotreeCmd.AddCommand(holotreePullCmd) + holotreePullCmd.Flags().BoolVarP(&forcePull, "force", "", false, "Force pull check, even when blueprint is already present.") + holotreePullCmd.Flags().StringVarP(&remoteOriginOption, "origin", "o", origin, "URL of remote origin to pull environment from.") + holotreePullCmd.Flags().StringVarP(&pullRobot, "robot", "r", "robot.yaml", "Full path to 'robot.yaml' configuration file to export as catalog. ") + if len(origin) == 0 { + holotreePullCmd.MarkFlagRequired("origin") + } +} diff --git a/cmd/holotreeRemove.go b/cmd/holotreeRemove.go new file mode 100644 index 00000000..01f5ab93 --- /dev/null +++ b/cmd/holotreeRemove.go @@ -0,0 +1,69 @@ +package cmd + +import ( + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/htfs" + "github.com/robocorp/rcc/pretty" + "github.com/spf13/cobra" +) + +var ( + removeCheckRetries int + unusedDays int +) + +func holotreeRemove(catalogs []string) { + if len(catalogs) == 0 { + pretty.Warning("No catalogs given, so nothing to do. Quitting!") + return + } + common.Debug("Trying to remove following catalogs:") + for _, catalog := range catalogs { + common.Debug("- %s", catalog) + } + + tree, err := htfs.New() + pretty.Guard(err == nil, 2, "%s", err) + + err = tree.Remove(catalogs) + pretty.Guard(err == nil, 3, "%s", err) +} + +func allUnusedCatalogs(limit int) []string { + result := []string{} + used := catalogUsedStats() + for name, idle := range used { + if idle > limit { + result = append(result, name) + } + } + return result +} + +var holotreeRemoveCmd = &cobra.Command{ + Use: "remove catalogid*", + Short: "Remove existing holotree catalogs.", + Long: "Remove existing holotree catalogs. Partial identities are ok.", + Aliases: []string{"rm"}, + Run: func(cmd *cobra.Command, args []string) { + if common.DebugFlag() { + defer common.Stopwatch("Holotree remove command lasted").Report() + } + if unusedDays > 0 { + args = append(args, allUnusedCatalogs(unusedDays)...) + } + holotreeRemove(selectCatalogs(args)) + if removeCheckRetries > 0 { + checkLoop(removeCheckRetries) + } else { + pretty.Warning("Remember to run `rcc holotree check` after you have removed all desired catalogs!") + } + pretty.Ok() + }, +} + +func init() { + holotreeRemoveCmd.Flags().IntVarP(&removeCheckRetries, "check", "c", 0, "Additionally run holotree check with this many times.") + holotreeRemoveCmd.Flags().IntVarP(&unusedDays, "unused", "", 0, "Remove idle/unused catalog entries based on idle days when value is above given limit.") + holotreeCmd.AddCommand(holotreeRemoveCmd) +} diff --git a/cmd/holotreeShared.go b/cmd/holotreeShared.go new file mode 100644 index 00000000..f381af20 --- /dev/null +++ b/cmd/holotreeShared.go @@ -0,0 +1,44 @@ +package cmd + +import ( + "os" + + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/pathlib" + "github.com/robocorp/rcc/pretty" + + "github.com/spf13/cobra" +) + +var ( + enableShared bool + onlyOnce bool +) + +var holotreeSharedCommand = &cobra.Command{ + Use: "shared", + Short: "Enable shared holotree usage.", + Long: "Enable shared holotree usage.", + Run: func(cmd *cobra.Command, args []string) { + if common.DebugFlag() { + defer common.Stopwatch("Enabling shared holotree lasted").Report() + } + enabled := pathlib.IsFile(common.SharedMarkerLocation()) + if enabled && onlyOnce { + pretty.Warning("Seems that sharing is already enabled! Quitting! [--once]") + pretty.Ok() + return + } + if os.Geteuid() > 0 { + pretty.Warning("Running this command might need sudo/root/elevated access rights. Still, trying ...") + } + osSpecificHolotreeSharing(enableShared) + pretty.Ok() + }, +} + +func init() { + holotreeSharedCommand.Flags().BoolVarP(&enableShared, "enable", "e", false, "Enable shared holotree environments between users. Currently cannot be undone.") + holotreeSharedCommand.Flags().BoolVarP(&onlyOnce, "once", "o", false, "Only try enabling if it has not been done yet.") + holotreeCmd.AddCommand(holotreeSharedCommand) +} diff --git a/cmd/holotreeStats.go b/cmd/holotreeStats.go new file mode 100644 index 00000000..5df8d637 --- /dev/null +++ b/cmd/holotreeStats.go @@ -0,0 +1,40 @@ +package cmd + +import ( + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/journal" + "github.com/robocorp/rcc/pretty" + + "github.com/spf13/cobra" +) + +var ( + onlyAssistantStats bool + onlyRobotStats bool + onlyPrepareStats bool + onlyVariablesStats bool + statsWeeks uint +) + +var holotreeStatsCmd = &cobra.Command{ + Use: "statistics", + Short: "Show holotree environment build and runtime statistics.", + Long: "Show holotree environment build and runtime statistics.", + Aliases: []string{"statistic", "stats", "stat", "st"}, + Run: func(cmd *cobra.Command, args []string) { + if common.DebugFlag() { + defer common.Stopwatch("Holotree stats calculation lasted").Report() + } + journal.ShowStatistics(statsWeeks, onlyAssistantStats, onlyRobotStats, onlyPrepareStats, onlyVariablesStats) + pretty.Ok() + }, +} + +func init() { + holotreeCmd.AddCommand(holotreeStatsCmd) + holotreeStatsCmd.Flags().BoolVarP(&onlyAssistantStats, "--assistants", "a", false, "Include 'assistant run' into stats.") + holotreeStatsCmd.Flags().BoolVarP(&onlyRobotStats, "--robots", "r", false, "Include 'robot run' into stats.") + holotreeStatsCmd.Flags().BoolVarP(&onlyPrepareStats, "--prepares", "p", false, "Include 'cloud prepare' into stats.") + holotreeStatsCmd.Flags().BoolVarP(&onlyVariablesStats, "--variables", "v", false, "Include 'holotree variables' into stats.") + holotreeStatsCmd.Flags().UintVarP(&statsWeeks, "--weeks", "w", 12, "Number of previous weeks to include into stats.") +} diff --git a/cmd/holotreeVariables.go b/cmd/holotreeVariables.go new file mode 100644 index 00000000..4033c27f --- /dev/null +++ b/cmd/holotreeVariables.go @@ -0,0 +1,181 @@ +package cmd + +import ( + "fmt" + "path/filepath" + "strings" + + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/conda" + "github.com/robocorp/rcc/htfs" + "github.com/robocorp/rcc/journal" + "github.com/robocorp/rcc/operations" + "github.com/robocorp/rcc/pathlib" + "github.com/robocorp/rcc/pretty" + "github.com/robocorp/rcc/robot" + "github.com/spf13/cobra" +) + +const ( + newEnvironment = `environment creation` +) + +var ( + holotreeBlueprint []byte + holotreeForce bool + holotreeJson bool +) + +func asSimpleMap(line string) map[string]string { + parts := strings.SplitN(strings.TrimSpace(line), "=", 2) + if len(parts) != 2 { + return nil + } + if len(parts[0]) == 0 { + return nil + } + result := make(map[string]string) + result["key"] = parts[0] + result["value"] = parts[1] + return result +} + +func asJson(items []string) error { + result := make([]map[string]string, 0, len(items)) + for _, line := range items { + entry := asSimpleMap(line) + if entry != nil { + result = append(result, entry) + } + } + content, err := operations.NiceJsonOutput(result) + if err != nil { + return err + } + common.Stdout("%s\n", content) + return nil +} + +func asExportedText(items []string) { + prefix := "export" + if conda.IsWindows() { + prefix = "SET" + } + for _, line := range items { + common.Stdout("%s %s\n", prefix, line) + } +} + +func holotreeExpandEnvironment(userFiles []string, packfile, environment, workspace string, validity int, force bool) []string { + var extra []string + var data operations.Token + common.TimelineBegin("environment expansion start") + defer common.TimelineEnd() + + config, holotreeBlueprint, err := htfs.ComposeFinalBlueprint(userFiles, packfile) + pretty.Guard(err == nil, 5, "%s", err) + + condafile := filepath.Join(common.ProductTemp(), common.BlueprintHash(holotreeBlueprint)) + err = pathlib.WriteFile(condafile, holotreeBlueprint, 0o644) + pretty.Guard(err == nil, 6, "%s", err) + + holozip := "" + if config != nil { + holozip = config.Holozip() + } + path, _, err := htfs.NewEnvironment(condafile, holozip, true, force, operations.PullCatalog) + if !common.WarrantyVoided() { + pretty.RccPointOfView(newEnvironment, err) + } + pretty.Guard(err == nil, 6, "%s", err) + + if Has(environment) { + common.Timeline("load robot environment") + developmentEnvironment, err := robot.LoadEnvironmentSetup(environment) + if err == nil { + extra = developmentEnvironment.AsEnvironment() + } + } + + common.Timeline("load robot environment") + var env []string + if config != nil { + env = config.RobotExecutionEnvironment(path, extra, false) + } else { + env = conda.CondaExecutionEnvironment(path, extra, false) + } + + if Has(workspace) { + common.Timeline("get run robot claims") + period := &operations.TokenPeriod{ + ValidityTime: validityTime, + GracePeriod: gracePeriod, + } + period.EnforceGracePeriod() + claims := operations.RunRobotClaims(period.RequestSeconds(), workspace) + data, err = operations.AuthorizeClaims(AccountName(), claims, period) + pretty.Guard(err == nil, 9, "Failed to get cloud data, reason: %v", err) + } + + if len(data) > 0 { + endpoint := data["endpoint"] + for _, key := range rcHosts { + env = append(env, fmt.Sprintf("%s=%s", key, endpoint)) + } + token := data["token"] + for _, key := range rcTokens { + env = append(env, fmt.Sprintf("%s=%s", key, token)) + } + env = append(env, fmt.Sprintf("RC_WORKSPACE_ID=%s", workspaceId)) + } + + return removeUnwanted(env) +} + +func removeUnwanted(variables []string) []string { + result := make([]string, 0, len(variables)) + for _, line := range variables { + switch { + case strings.HasPrefix(line, "PS1="): + continue + default: + result = append(result, line) + } + } + + return result +} + +var holotreeVariablesCmd = &cobra.Command{ + Use: "variables conda.yaml+", + Aliases: []string{"vars"}, + Short: "Do holotree operations.", + Long: "Do holotree operations.", + Run: func(cmd *cobra.Command, args []string) { + defer journal.BuildEventStats("variables") + if common.DebugFlag() { + defer common.Stopwatch("Holotree variables command lasted").Report() + } + + env := holotreeExpandEnvironment(args, robotFile, environmentFile, workspaceId, validityTime, holotreeForce) + if holotreeJson { + asJson(env) + } else { + asExportedText(env) + } + }, +} + +func init() { + holotreeCmd.AddCommand(holotreeVariablesCmd) + holotreeVariablesCmd.Flags().StringVarP(&environmentFile, "environment", "e", "", "Full path to 'env.json' development environment data file. ") + holotreeVariablesCmd.Flags().StringVarP(&robotFile, "robot", "r", "robot.yaml", "Full path to 'robot.yaml' configuration file. ") + holotreeVariablesCmd.Flags().StringVarP(&workspaceId, "workspace", "w", "", "Optional workspace id to get authorization tokens for. ") + holotreeVariablesCmd.Flags().IntVarP(&validityTime, "minutes", "m", 15, "How many minutes the authorization should be valid for (minimum 15 minutes).") + holotreeVariablesCmd.Flags().IntVarP(&gracePeriod, "graceperiod", "", 5, "What is grace period buffer in minutes on top of validity minutes (minimum 5 minutes).") + holotreeVariablesCmd.Flags().StringVarP(&accountName, "account", "a", "", "Account used for workspace. ") + + holotreeVariablesCmd.Flags().StringVarP(&common.HolotreeSpace, "space", "s", "user", "Client specific name to identify this environment.") + holotreeVariablesCmd.Flags().BoolVarP(&holotreeForce, "force", "f", false, "Force environment creation with refresh.") + holotreeVariablesCmd.Flags().BoolVarP(&holotreeJson, "json", "j", false, "Show environment as JSON.") +} diff --git a/cmd/holotreeVenv.go b/cmd/holotreeVenv.go new file mode 100644 index 00000000..742a3cf3 --- /dev/null +++ b/cmd/holotreeVenv.go @@ -0,0 +1,116 @@ +package cmd + +import ( + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" + + "github.com/robocorp/rcc/blobs" + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/conda" + "github.com/robocorp/rcc/fail" + "github.com/robocorp/rcc/htfs" + "github.com/robocorp/rcc/journal" + "github.com/robocorp/rcc/pathlib" + "github.com/robocorp/rcc/pretty" + "github.com/robocorp/rcc/shell" + + "github.com/spf13/cobra" +) + +func deleteByExactIdentity(exact string) { + _, roots := htfs.LoadCatalogs() + for _, label := range roots.FindEnvironments([]string{exact}) { + common.Log("Removing %v", label) + err := roots.RemoveHolotreeSpace(label) + pretty.Guard(err == nil, 4, "Error: %v", err) + } +} + +var holotreeVenvCmd = &cobra.Command{ + Use: "venv conda.yaml+", + Short: "Create user managed virtual python environment inside automation folder.", + Long: "Create user managed virtual python environment inside automation folder.", + Args: cobra.MinimumNArgs(1), + + Run: func(cmd *cobra.Command, args []string) { + defer journal.BuildEventStats("venv") + if common.DebugFlag() { + defer common.Stopwatch("Holotree venv command lasted").Report() + } + + // following settings are forced in venv environments + common.UnmanagedSpace = true + common.ExternallyManaged = true + common.ControllerType = "venv" + + where, err := os.Getwd() + pretty.Guard(err == nil, 1, "Error: %v", err) + location := filepath.Join(where, "venv") + + previous := pathlib.IsDir(location) + if holotreeForce && previous { + pretty.Note("Trying to remove existing venv at %q ...", location) + err := pathlib.TryRemoveAll("venv", location) + pretty.Guard(err == nil, 2, "Error: %v", err) + } + + pretty.Guard(!pathlib.Exists(location), 3, "Name %q aready exists! Remove it, or use force.", location) + + if holotreeForce { + identity := htfs.ControllerSpaceName([]byte(common.ControllerIdentity()), []byte(common.HolotreeSpace)) + deleteByExactIdentity(identity) + } + + env := holotreeExpandEnvironment(args, "", "", "", 0, holotreeForce) + envPath := pathlib.EnvironmentPath(env) + python, ok := envPath.Which("python", conda.FileExtensions) + if !ok { + python, ok = envPath.Which("python3", conda.FileExtensions) + } + pretty.Guard(ok, 5, "For some reason, could not find python executable in environment paths. Report a bug. PATH: %q", envPath) + pretty.Note("Trying to make new venv at %q using %q ...", location, python) + task := shell.New(env, ".", python, "-m", "venv", "--system-site-packages", location) + code, err := task.Execute(false) + pretty.Guard(err == nil, 6, "Error: %v", err) + pretty.Guard(code == 0, 7, "Exit code %d from venv creation.", code) + + target := listActivationScripts(location) + if len(target) > 0 { + blob, err := blobs.Asset("assets/depxtraction.py") + fail.Fast(err) + location := filepath.Join(target, "depxtraction.py") + fail.Fast(os.WriteFile(location, blob, 0o755)) + fmt.Printf("Experimental dependency extraction tool is available at %q.\nTry it after pip installing things into your venv.\n", location) + } + + pretty.Ok() + }, +} + +func listActivationScripts(root string) string { + pretty.Highlight("New venv is located at %s. Following scripts seem to be available:", root) + base := filepath.Dir(root) + pathcandidate := "" + filepath.Walk(root, func(path string, entry fs.FileInfo, err error) error { + if entry.Mode().IsRegular() && strings.HasPrefix(strings.ToLower(entry.Name()), "activ") { + short, err := filepath.Rel(base, path) + if err == nil { + pretty.Highlight(" - %s", short) + } + pathcandidate = filepath.Dir(short) + } + return nil + }) + return pathcandidate +} + +func init() { + rootCmd.AddCommand(holotreeVenvCmd) + holotreeCmd.AddCommand(holotreeVenvCmd) + + holotreeVenvCmd.Flags().StringVarP(&common.HolotreeSpace, "space", "s", "user", "Client specific name to identify this environment.") + holotreeVenvCmd.Flags().BoolVarP(&holotreeForce, "force", "f", false, "Force environment creation by deleting unmanaged space. Dangerous, do not use unless you understand what it means.") +} diff --git a/cmd/initialize.go b/cmd/initialize.go index bd5b9456..ec12e339 100644 --- a/cmd/initialize.go +++ b/cmd/initialize.go @@ -8,19 +8,30 @@ import ( "github.com/spf13/cobra" ) +var ( + internalOnlyFlag bool +) + func createWorkarea() { - if len(directory) == 0 { - pretty.Exit(1, "Error: missing target directory") - } - err := operations.InitializeWorkarea(directory, templateName, forceFlag) - if err != nil { - pretty.Exit(2, "Error: %v", err) + pretty.Guard(len(directory) > 0, 1, "Error: missing target directory (option: --directory)") + pretty.Guard(len(templateName) > 0, 3, "Error: missing template name (option: --template)") + err := operations.InitializeWorkarea(directory, templateName, internalOnlyFlag, forceFlag) + pretty.Guard(err == nil, 2, "Error: %v", err) +} + +func listJsonTemplates() { + templates := make(map[string]string) + for _, pair := range operations.ListTemplatesWithDescription(internalOnlyFlag) { + templates[pair[0]] = pair[1] } + out, err := operations.NiceJsonOutput(templates) + pretty.Guard(err == nil, 2, "Failed to format templates as JSON, reason: %s", err) + common.Stdout("%s\n", out) } func listTemplates() { common.Stdout("Template names:\n") - for _, name := range operations.ListTemplates() { + for _, name := range operations.ListTemplates(internalOnlyFlag) { common.Stdout("- %v\n", name) } } @@ -31,9 +42,13 @@ var initializeCmd = &cobra.Command{ Short: "Create a directory structure for a robot.", Long: "Create a directory structure for a robot.", Run: func(cmd *cobra.Command, args []string) { - if common.DebugFlag { + if common.DebugFlag() { defer common.Stopwatch("Initialization lasted").Report() } + if jsonFlag { + listJsonTemplates() + return + } if listFlag { listTemplates() } else { @@ -46,7 +61,9 @@ var initializeCmd = &cobra.Command{ func init() { robotCmd.AddCommand(initializeCmd) initializeCmd.Flags().StringVarP(&directory, "directory", "d", ".", "Root directory to create the new robot in.") - initializeCmd.Flags().StringVarP(&templateName, "template", "t", "standard", "Template to use to generate the robot content.") + initializeCmd.Flags().StringVarP(&templateName, "template", "t", "", "Template to use to generate the robot content.") initializeCmd.Flags().BoolVarP(&forceFlag, "force", "f", false, "Force the creation of the robot and possibly overwrite data.") initializeCmd.Flags().BoolVarP(&listFlag, "list", "l", false, "List available templates.") + initializeCmd.Flags().BoolVarP(&jsonFlag, "json", "j", false, "List available templates as JSON.") + initializeCmd.Flags().BoolVarP(&internalOnlyFlag, "internal", "i", false, "Undocumented. Deprecated. DO NOT USE.") } diff --git a/cmd/install.go b/cmd/install.go deleted file mode 100644 index 334868ec..00000000 --- a/cmd/install.go +++ /dev/null @@ -1,26 +0,0 @@ -package cmd - -import ( - "github.com/robocorp/rcc/conda" - "github.com/robocorp/rcc/pretty" - - "github.com/spf13/cobra" -) - -var installCmd = &cobra.Command{ - Use: "install", - Aliases: []string{"i"}, - Short: "Install miniconda into the managed location.", - Long: `Install miniconda into the rcc managed location. Before executing this command, -you must successfully run the "download" command and verify that the miniconda SHA256 -matches the one on the conda site.`, - Run: func(cmd *cobra.Command, args []string) { - if !conda.DoInstall() { - pretty.Exit(1, "Error: Install failed. See above.") - } - }, -} - -func init() { - condaCmd.AddCommand(installCmd) -} diff --git a/cmd/interactive.go b/cmd/interactive.go index 0b4cd425..60b7fa80 100644 --- a/cmd/interactive.go +++ b/cmd/interactive.go @@ -1,6 +1,7 @@ package cmd import ( + "github.com/robocorp/rcc/common" "github.com/spf13/cobra" ) @@ -13,5 +14,7 @@ Do not try to use these in automation, they will fail there.`, } func init() { - rootCmd.AddCommand(interactiveCmd) + if common.Product.IsLegacy() { + rootCmd.AddCommand(interactiveCmd) + } } diff --git a/cmd/internal.go b/cmd/internal.go index 9ef329f8..a93861cf 100644 --- a/cmd/internal.go +++ b/cmd/internal.go @@ -1,6 +1,7 @@ package cmd import ( + "github.com/robocorp/rcc/common" "github.com/spf13/cobra" ) @@ -13,5 +14,8 @@ var internalCmd = &cobra.Command{ } func init() { - rootCmd.AddCommand(internalCmd) + if common.Product.IsLegacy() { + rootCmd.AddCommand(internalCmd) + internalCmd.PersistentFlags().StringVarP(&wskey, "wskey", "", "", "Cloud API workspace key (authorization).") + } } diff --git a/cmd/internalEnv.go b/cmd/internalEnv.go new file mode 100644 index 00000000..936c5717 --- /dev/null +++ b/cmd/internalEnv.go @@ -0,0 +1,38 @@ +package cmd + +import ( + "encoding/json" + "fmt" + "os" + "strings" + + "github.com/robocorp/rcc/pretty" + "github.com/spf13/cobra" +) + +var preformatLabel string + +var internalEnvCmd = &cobra.Command{ + Use: "env", + Short: "JSON dump of current execution environment.", + Long: "JSON dump of current execution environment.", + Run: func(cmd *cobra.Command, args []string) { + values := make(map[string]string) + for _, entry := range os.Environ() { + parts := strings.SplitN(entry, "=", 2) + if len(parts) == 2 && len(parts[0]) > 0 { + values[parts[0]] = parts[1] + } + } + result, err := json.MarshalIndent(values, "", " ") + pretty.Guard(err == nil, 1, "Fail: %v", err) + + fmt.Fprintf(os.Stdout, "``` env dump %q begins\n%s\n```\n", preformatLabel, result) + }, +} + +func init() { + internalCmd.AddCommand(internalEnvCmd) + internalEnvCmd.Flags().StringVarP(&preformatLabel, "label", "l", "", "Label to identitfy variable dump.") + internalEnvCmd.MarkFlagRequired("label") +} diff --git a/cmd/internale2ee.go b/cmd/internale2ee.go deleted file mode 100644 index 637d3de9..00000000 --- a/cmd/internale2ee.go +++ /dev/null @@ -1,54 +0,0 @@ -package cmd - -import ( - "github.com/robocorp/rcc/cloud" - "github.com/robocorp/rcc/common" - "github.com/robocorp/rcc/operations" - "github.com/robocorp/rcc/pretty" - - "github.com/spf13/cobra" -) - -var e2eeCmd = &cobra.Command{ - Use: "encryption", - Short: "Internal end-to-end encryption tester method", - Long: "Internal end-to-end encryption tester method", - Args: cobra.ExactArgs(1), - Run: func(cmd *cobra.Command, args []string) { - if common.DebugFlag { - defer common.Stopwatch("Encryption lasted").Report() - } - account := operations.AccountByName(AccountName()) - if account == nil { - pretty.Exit(1, "Could not find account by name: %v", AccountName()) - } - client, err := cloud.NewClient(account.Endpoint) - if err != nil { - pretty.Exit(2, "Could not create client for endpoint: %v, reason: %v", account.Endpoint, err) - } - key, err := operations.GenerateEphemeralKey() - if err != nil { - pretty.Exit(3, "Problem with key generation, reason: %v", err) - } - request := client.NewRequest("/assistant-v1/test/encryption") - request.Body, err = key.RequestBody(args[0]) - if err != nil { - pretty.Exit(4, "Problem with body generation, reason: %v", err) - } - response := client.Post(request) - if response.Status != 200 { - pretty.Exit(5, "Problem with test request, status=%d, body=%s", response.Status, response.Body) - } - plaintext, err := key.Decode(response.Body) - if err != nil { - pretty.Exit(6, "Decode problem with body %s, reason: %v", response.Body, err) - } - common.Log("Response: %s", string(plaintext)) - pretty.Ok() - }, -} - -func init() { - internalCmd.AddCommand(e2eeCmd) - e2eeCmd.Flags().StringVarP(&accountName, "account", "a", "", "Account used for Robocorp Cloud operations.") -} diff --git a/cmd/libs.go b/cmd/libs.go deleted file mode 100644 index 41cd65fd..00000000 --- a/cmd/libs.go +++ /dev/null @@ -1,58 +0,0 @@ -package cmd - -import ( - "github.com/robocorp/rcc/common" - "github.com/robocorp/rcc/conda" - "github.com/robocorp/rcc/pretty" - - "github.com/spf13/cobra" -) - -var ( - channelFlag bool - pipFlag bool - dryFlag bool - - condaOption string - nameOption string - addMany []string - removeMany []string -) - -var libsCmd = &cobra.Command{ - Use: "libs", - Aliases: []string{"library", "libraries"}, - Short: "Manage library dependencies in an action oriented way.", - Long: "Manage library dependencies in an action oriented way.", - Run: func(cmd *cobra.Command, args []string) { - if common.DebugFlag { - defer common.Stopwatch("Robot libs lasted").Report() - } - changes := &conda.Changes{ - Name: nameOption, - Pip: pipFlag, - Dryrun: dryFlag, - Channel: channelFlag, - Add: addMany, - Remove: removeMany, - } - output, err := conda.UpdateEnvironment(condaOption, changes) - if err != nil { - pretty.Exit(1, "Error: %v", err) - } - common.Stdout("%s\n", output) - pretty.Ok() - }, -} - -func init() { - robotCmd.AddCommand(libsCmd) - libsCmd.Flags().StringVarP(&nameOption, "name", "n", "", "Change the name of the configuration.") - libsCmd.Flags().StringVarP(&condaOption, "conda", "", "", "Full path to the conda environment configuration file (conda.yaml).") - libsCmd.MarkFlagRequired("conda") - libsCmd.Flags().StringArrayVarP(&addMany, "add", "a", []string{}, "Add new libraries as requirements.") - libsCmd.Flags().StringArrayVarP(&removeMany, "remove", "r", []string{}, "Remove existing libraries from requirements.") - libsCmd.Flags().BoolVarP(&channelFlag, "channel", "c", false, "Operate on channels (default is packages).") - libsCmd.Flags().BoolVarP(&pipFlag, "pip", "p", false, "Operate on pip packages (the default is to operate on conda packages).") - libsCmd.Flags().BoolVarP(&dryFlag, "dryrun", "d", false, "Do not save the end result, just show what would happen.") -} diff --git a/cmd/license.go b/cmd/license.go deleted file mode 100644 index c6c959e9..00000000 --- a/cmd/license.go +++ /dev/null @@ -1,26 +0,0 @@ -package cmd - -import ( - "github.com/robocorp/rcc/blobs" - "github.com/robocorp/rcc/common" - "github.com/robocorp/rcc/pretty" - - "github.com/spf13/cobra" -) - -var licenseCmd = &cobra.Command{ - Use: "license", - Short: "Show the rcc License.", - Long: "Show the rcc License.", - Run: func(cmd *cobra.Command, args []string) { - content, err := blobs.Asset("assets/man/LICENSE.txt") - if err != nil { - pretty.Exit(1, "Cannot show LICENSE, reason: %v", err) - } - common.Stdout("%s\n", content) - }, -} - -func init() { - manCmd.AddCommand(licenseCmd) -} diff --git a/cmd/list.go b/cmd/list.go deleted file mode 100644 index 3b379f69..00000000 --- a/cmd/list.go +++ /dev/null @@ -1,49 +0,0 @@ -package cmd - -import ( - "fmt" - "sort" - "time" - - "github.com/robocorp/rcc/common" - "github.com/robocorp/rcc/conda" - "github.com/robocorp/rcc/pretty" - - "github.com/spf13/cobra" -) - -var listCmd = &cobra.Command{ - Use: "list", - Short: "Listing currently managed virtual environments.", - Long: `List shows listing of currently managed virtual environments -in human readable form.`, - Run: func(cmd *cobra.Command, args []string) { - templates := conda.TemplateList() - if len(templates) == 0 { - pretty.Exit(1, "No environments available.") - } - lines := make([]string, 0, len(templates)) - common.Log("%-25s %-25s %s", "Last used", "Last cloned", "Environment (TLSH)") - for _, template := range templates { - cloned := "N/A" - used := cloned - when, err := conda.LastUsed(conda.TemplateFrom(template)) - if err == nil { - cloned = when.Format(time.RFC3339) - } - when, err = conda.LastUsed(conda.LiveFrom(template)) - if err == nil { - used = when.Format(time.RFC3339) - } - lines = append(lines, fmt.Sprintf("%-25s %-25s %s", used, cloned, template)) - } - sort.Strings(lines) - for _, line := range lines { - common.Log("%s", line) - } - }, -} - -func init() { - envCmd.AddCommand(listCmd) -} diff --git a/cmd/longpaths.go b/cmd/longpaths.go new file mode 100644 index 00000000..70c24ea7 --- /dev/null +++ b/cmd/longpaths.go @@ -0,0 +1,39 @@ +package cmd + +import ( + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/conda" + "github.com/robocorp/rcc/pretty" + "github.com/spf13/cobra" +) + +var ( + enableLongpaths bool +) + +var longpathsCmd = &cobra.Command{ + Use: "longpaths", + Short: "Check and enable Windows longpath support", + Long: "Check and enable Windows longpath support", + Run: func(cmd *cobra.Command, args []string) { + if common.OverrideSystemRequirements() { + pretty.Exit(100, "This operation is prevented, because ROBOCORP_OVERRIDE_SYSTEM_REQUIREMENTS is effective!") + } + var err error + if enableLongpaths { + err = conda.EnforceLongpathSupport() + } + if err != nil { + pretty.Exit(1, "Failure to modify registry: %v", err) + } + if !conda.HasLongPathSupport() { + pretty.Exit(2, "Long paths do not work!") + } + pretty.Ok() + }, +} + +func init() { + configureCmd.AddCommand(longpathsCmd) + longpathsCmd.Flags().BoolVarP(&enableLongpaths, "enable", "e", false, "Change registry settings and enable longpath support") +} diff --git a/cmd/lsh.go b/cmd/lsh.go deleted file mode 100644 index b5f1d5fb..00000000 --- a/cmd/lsh.go +++ /dev/null @@ -1,41 +0,0 @@ -package cmd - -import ( - "github.com/robocorp/rcc/common" - "github.com/robocorp/rcc/conda" - "github.com/robocorp/rcc/pretty" - - "github.com/spf13/cobra" -) - -var lshCmd = &cobra.Command{ - Use: "lsh", - Short: "Locality-sensitive hash calculation", - Long: `This lsh command calculates locality-sensitive hash from environment.yaml, -or requirements.txt, or similar text files.`, - Args: cobra.MinimumNArgs(1), - Run: func(cmd *cobra.Command, args []string) { - baseline := "" - failure := false - for _, arg := range args { - out, err := conda.HashConfig(arg) - if err != nil { - common.Error("lsh", err) - failure = true - continue - } - if baseline == "" { - baseline = out - } - distance, _ := conda.Distance(baseline, out) - common.Log("%s: %s <%d>", out, arg, distance) - } - if failure { - pretty.Exit(1, "Error!") - } - }, -} - -func init() { - internalCmd.AddCommand(lshCmd) -} diff --git a/cmd/man.go b/cmd/man.go index 89f379c8..58956b66 100644 --- a/cmd/man.go +++ b/cmd/man.go @@ -1,9 +1,16 @@ package cmd import ( + "github.com/robocorp/rcc/blobs" + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/pretty" "github.com/spf13/cobra" ) +type ( + cobraCommand func(*cobra.Command, []string) +) + var manCmd = &cobra.Command{ Use: "man", Aliases: []string{"manuals", "docs", "doc", "guides", "guide", "m"}, @@ -13,4 +20,100 @@ var manCmd = &cobra.Command{ func init() { rootCmd.AddCommand(manCmd) + + manCmd.AddCommand(&cobra.Command{ + Use: "changelog", + Short: "Show the rcc changelog.", + Long: "Show the rcc changelog.", + Aliases: []string{"changes"}, + Run: makeShowDoc("changelog", "docs/changelog.md"), + }) + + manCmd.AddCommand(&cobra.Command{ + Use: "features", + Short: "Show some of rcc features.", + Long: "Show some of rcc features.", + Run: makeShowDoc("features", "docs/features.md"), + }) + + manCmd.AddCommand(&cobra.Command{ + Use: "license", + Short: "Show the rcc License.", + Long: "Show the rcc License.", + Run: makeShowDoc("LICENSE", "assets/man/LICENSE.txt"), + }) + + manCmd.AddCommand(&cobra.Command{ + Use: "maintenance", + Short: "Show holotree maintenance documentation.", + Long: "Show holotree maintenance documentation.", + Run: makeShowDoc("holotree maintenance documentation", "docs/maintenance.md"), + }) + + manCmd.AddCommand(&cobra.Command{ + Use: "profiles", + Short: "Show configuration profiles documentation.", + Long: "Show configuration profiles documentation.", + Run: makeShowDoc("profile documentation", "docs/profile_configuration.md"), + }) + + manCmd.AddCommand(&cobra.Command{ + Use: "recipes", + Short: "Show rcc recipes, tips, and tricks.", + Long: "Show rcc recipes, tips, and tricks.", + Aliases: []string{"recipe", "tips", "tricks"}, + Run: makeShowDoc("recipes", "docs/recipes.md"), + }) + + manCmd.AddCommand(&cobra.Command{ + Use: "troubleshooting", + Short: "Show the rcc troubleshooting documentation.", + Long: "Show the rcc troubleshooting documentation.", + Run: makeShowDoc("troubleshooting", "docs/troubleshooting.md"), + }) + + manCmd.AddCommand(&cobra.Command{ + Use: "usecases", + Short: "Show some of rcc use cases.", + Long: "Show some of rcc use cases.", + Run: makeShowDoc("use-cases", "docs/usecases.md"), + }) + + tutorial := &cobra.Command{ + Use: "tutorial", + Short: "Show the rcc tutorial.", + Long: "Show the rcc tutorial.", + Aliases: []string{"tut"}, + Run: makeShowDoc("tutorial", "assets/man/tutorial.txt"), + } + + manCmd.AddCommand(&cobra.Command{ + Use: "venv", + Short: "Show virtual environment documentation.", + Long: "Show virtual environment documentation.", + Run: makeShowDoc("venv", "docs/venv.md"), + }) + + manCmd.AddCommand(&cobra.Command{ + Use: "vocabulary", + Short: "Show vocabulary documentation", + Long: "Show vocabulary documentation", + Aliases: []string{"glossary", "lexicon"}, + Run: makeShowDoc("vocabulary documentation", "docs/vocabulary.md"), + }) + + if common.Product.IsLegacy() { + manCmd.AddCommand(tutorial) + rootCmd.AddCommand(tutorial) + } +} + +func makeShowDoc(label, asset string) cobraCommand { + return func(cmd *cobra.Command, args []string) { + content, err := blobs.Asset(asset) + if err != nil { + pretty.Exit(1, "Cannot show %s documentation, reason: %v", label, err) + } + pretty.Page(content) + } } diff --git a/cmd/merge.go b/cmd/merge.go index db81669b..31668e11 100644 --- a/cmd/merge.go +++ b/cmd/merge.go @@ -27,7 +27,7 @@ var mergeCmd = &cobra.Command{ for _, filename := range args { left = right - right, err = conda.ReadCondaYaml(filename) + right, err = conda.ReadPackageCondaYaml(filename) if err != nil { pretty.Exit(1, err.Error()) } diff --git a/cmd/metric.go b/cmd/metric.go deleted file mode 100644 index d4396088..00000000 --- a/cmd/metric.go +++ /dev/null @@ -1,42 +0,0 @@ -package cmd - -import ( - "github.com/robocorp/rcc/common" - "github.com/robocorp/rcc/operations" - "github.com/robocorp/rcc/pretty" - "github.com/robocorp/rcc/xviper" - - "github.com/spf13/cobra" -) - -var ( - metricType string - metricName string - metricValue string -) - -var metricCmd = &cobra.Command{ - Use: "metric", - Short: "Send some metric to Robocorp Cloud.", - Long: "Send some metric to Robocorp Cloud.", - Run: func(cmd *cobra.Command, args []string) { - if common.DebugFlag { - defer common.Stopwatch("Feedback metric lasted").Report() - } - if !xviper.CanTrack() { - pretty.Exit(1, "Tracking is disabled. Quitting.") - } - operations.SendMetric(metricType, metricName, metricValue) - pretty.Exit(0, "OK") - }, -} - -func init() { - feedbackCmd.AddCommand(metricCmd) - metricCmd.Flags().StringVarP(&metricType, "type", "t", "", "Type for metric source to use.") - metricCmd.MarkFlagRequired("type") - metricCmd.Flags().StringVarP(&metricName, "name", "n", "", "Name for metric to report.") - metricCmd.MarkFlagRequired("name") - metricCmd.Flags().StringVarP(&metricValue, "value", "v", "", "Value for metric to report.") - metricCmd.MarkFlagRequired("value") -} diff --git a/cmd/netdiagnostics.go b/cmd/netdiagnostics.go new file mode 100644 index 00000000..c746e67f --- /dev/null +++ b/cmd/netdiagnostics.go @@ -0,0 +1,58 @@ +package cmd + +import ( + "os" + + "github.com/robocorp/rcc/blobs" + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/operations" + "github.com/robocorp/rcc/pretty" + + "github.com/spf13/cobra" +) + +var ( + netConfigFilename string + netConfigShow bool +) + +func summonNetworkDiagConfig(filename string) ([]byte, error) { + if len(filename) == 0 { + return blobs.Asset("assets/netdiag.yaml") + } + return os.ReadFile(filename) +} + +var netDiagnosticsCmd = &cobra.Command{ + Use: "netdiagnostics", + Aliases: []string{"netdiagnostic", "netdiag"}, + Short: "Run additional diagnostics to help resolve network issues.", + Long: "Run additional diagnostics to help resolve network issues.", + Run: func(cmd *cobra.Command, args []string) { + if common.DebugFlag() { + defer common.Stopwatch("Netdiagnostic run lasted").Report() + } + config, err := summonNetworkDiagConfig(netConfigFilename) + if err != nil { + pretty.Exit(1, "Problem loading configuration file, reason: %v", err) + } + if netConfigShow { + common.Stdout("%s", string(config)) + os.Exit(0) + } + _, err = operations.ProduceNetDiagnostics(config, jsonFlag) + if err != nil { + pretty.Exit(1, "Error: %v", err) + } + pretty.Ok() + }, +} + +func init() { + configureCmd.AddCommand(netDiagnosticsCmd) + rootCmd.AddCommand(netDiagnosticsCmd) + + netDiagnosticsCmd.Flags().BoolVarP(&jsonFlag, "json", "j", false, "Output in JSON format") + netDiagnosticsCmd.Flags().BoolVarP(&netConfigShow, "show", "s", false, "Show configuration instead of running diagnostics.") + netDiagnosticsCmd.Flags().StringVarP(&netConfigFilename, "checks", "c", "", "Network checks configuration file. [optional]") +} diff --git a/cmd/pull.go b/cmd/pull.go index eab61860..e7cc800f 100644 --- a/cmd/pull.go +++ b/cmd/pull.go @@ -4,11 +4,11 @@ import ( "fmt" "os" "path/filepath" - "time" "github.com/robocorp/rcc/cloud" "github.com/robocorp/rcc/common" "github.com/robocorp/rcc/operations" + "github.com/robocorp/rcc/pathlib" "github.com/robocorp/rcc/pretty" "github.com/spf13/cobra" @@ -16,16 +16,16 @@ import ( var pullCmd = &cobra.Command{ Use: "pull", - Short: "Pull a robot from Robocorp Cloud and unwrap it into local directory.", - Long: "Pull a robot from Robocorp Cloud and unwrap it into local directory.", + Short: fmt.Sprintf("Pull a robot from %s Control Room and unwrap it into local directory.", common.Product.Name()), + Long: fmt.Sprintf("Pull a robot from %s Control Room and unwrap it into local directory.", common.Product.Name()), Run: func(cmd *cobra.Command, args []string) { - if common.DebugFlag { + if common.DebugFlag() { defer common.Stopwatch("Pull lasted").Report() } account := operations.AccountByName(AccountName()) if account == nil { - pretty.Exit(1, "Could not find account by name: %v", AccountName()) + pretty.Exit(1, "Could not find account by name: %q", AccountName()) } client, err := cloud.NewClient(account.Endpoint) @@ -33,16 +33,16 @@ var pullCmd = &cobra.Command{ pretty.Exit(2, "Could not create client for endpoint: %v reason %v", account.Endpoint, err) } - zipfile := filepath.Join(os.TempDir(), fmt.Sprintf("pull%x.zip", time.Now().Unix())) + zipfile := filepath.Join(pathlib.TempDir(), fmt.Sprintf("pull%x.zip", common.When)) defer os.Remove(zipfile) common.Debug("Using temporary zipfile at %v", zipfile) - err = operations.DownloadCommand(client, account, workspaceId, robotId, zipfile, common.DebugFlag) + err = operations.DownloadCommand(client, account, workspaceId, robotId, zipfile, common.DebugFlag()) if err != nil { pretty.Exit(3, "Error: %v", err) } - err = operations.Unzip(directory, zipfile, forceFlag, false) + err = operations.Unzip(directory, zipfile, forceFlag, false, true) if err != nil { pretty.Exit(4, "Error: %v", err) } diff --git a/cmd/push.go b/cmd/push.go index 566d96c3..798a9052 100644 --- a/cmd/push.go +++ b/cmd/push.go @@ -4,11 +4,11 @@ import ( "fmt" "os" "path/filepath" - "time" "github.com/robocorp/rcc/cloud" "github.com/robocorp/rcc/common" "github.com/robocorp/rcc/operations" + "github.com/robocorp/rcc/pathlib" "github.com/robocorp/rcc/pretty" "github.com/spf13/cobra" @@ -16,22 +16,22 @@ import ( var pushCmd = &cobra.Command{ Use: "push", - Short: "Wrap the local directory and push it into Robocorp Cloud as a specific robot.", - Long: "Wrap the local directory and push it into Robocorp Cloud as a specific robot.", + Short: fmt.Sprintf("Wrap the local directory and push it into %s Control Room as a specific robot.", common.Product.Name()), + Long: fmt.Sprintf("Wrap the local directory and push it into %s Control Room as a specific robot.", common.Product.Name()), Run: func(cmd *cobra.Command, args []string) { - if common.DebugFlag { + if common.DebugFlag() { defer common.Stopwatch("Push lasted").Report() } account := operations.AccountByName(AccountName()) if account == nil { - pretty.Exit(1, "Could not find account by name: %v", AccountName()) + pretty.Exit(1, "Could not find account by name: %q", AccountName()) } client, err := cloud.NewClient(account.Endpoint) if err != nil { pretty.Exit(2, "Could not create client for endpoint: %v reason: %v", account.Endpoint, err) } - zipfile := filepath.Join(os.TempDir(), fmt.Sprintf("push%x.zip", time.Now().Unix())) + zipfile := filepath.Join(pathlib.TempDir(), fmt.Sprintf("push%x.zip", common.When)) defer os.Remove(zipfile) common.Debug("Using temporary zipfile at %v", zipfile) @@ -39,11 +39,11 @@ var pushCmd = &cobra.Command{ if err != nil { pretty.Exit(3, "Error: %v", err) } - err = operations.UploadCommand(client, account, workspaceId, robotId, zipfile, common.DebugFlag) + err = operations.UploadCommand(client, account, workspaceId, robotId, zipfile, common.DebugFlag()) if err != nil { pretty.Exit(4, "Error: %v", err) } - operations.BackgroundMetric("rcc", "rcc.cli.push", common.Version) + cloud.InternalBackgroundMetric(common.ControllerIdentity(), "rcc.cli.push", common.Version) pretty.Ok() }, } diff --git a/cmd/rcc/main.go b/cmd/rcc/main.go index 0e17a0cf..96c9b7a7 100644 --- a/cmd/rcc/main.go +++ b/cmd/rcc/main.go @@ -1,29 +1,172 @@ package main import ( + "fmt" "os" + "os/user" + "path/filepath" + "runtime" + "strings" + "time" + "github.com/robocorp/rcc/anywork" + "github.com/robocorp/rcc/cloud" "github.com/robocorp/rcc/cmd" "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/conda" "github.com/robocorp/rcc/operations" + "github.com/robocorp/rcc/pathlib" + "github.com/robocorp/rcc/pretty" + "github.com/robocorp/rcc/set" ) +const ( + timezonekey = `rcc.cli.tz` + oskey = `rcc.cli.os` + daily = 60 * 60 * 24 +) + +var ( + markedAlready = false +) + +func EnsureUserRegistered() (string, error) { + var warning string + + cache, err := operations.SummonCache() + if err != nil { + return warning, err + } + who, err := user.Current() + if err != nil { + return warning, err + } + updated, ok := set.Update(cache.Users, strings.ToLower(who.Username)) + size := len(updated) + if size > 1 { + warning = fmt.Sprintf("More than one user is using same %s location! Those users are: %s!", common.Product.HomeVariable(), strings.Join(updated, ", ")) + } + if !ok { + return warning, nil + } + cache.Users = updated + return warning, cache.Save() +} + +func TimezoneMetric() error { + cache, err := operations.SummonCache() + if err != nil { + return err + } + deadline, ok := cache.Stamps[timezonekey] + if ok && deadline > common.When { + return nil + } + cache.Stamps[timezonekey] = common.When + daily + zone := time.Now().Format("MST-0700") + cloud.InternalBackgroundMetric(common.ControllerIdentity(), timezonekey, zone) + cloud.InternalBackgroundMetric(common.ControllerIdentity(), oskey, common.Platform()) + return cache.Save() +} + func ExitProtection() { + runtime.Gosched() status := recover() if status != nil { + markTempForRecycling() exit, ok := status.(common.ExitCode) if ok { exit.ShowMessage() + pretty.Highlight("[rcc] exit status will be: %d!", exit.Code) + cloud.WaitTelemetry() + common.WaitLogs() os.Exit(exit.Code) } - operations.SendMetric("rcc", "rcc.panic.origin", cmd.Origin()) + cloud.InternalBackgroundMetric(common.ControllerIdentity(), "rcc.panic.origin", cmd.Origin()) + cloud.WaitTelemetry() + common.WaitLogs() panic(status) } + cloud.WaitTelemetry() + common.WaitLogs() +} + +func startTempRecycling() { + if common.DisableTempManagement() { + common.Timeline("temp management disabled -- no temp recycling") + return + } + defer common.Timeline("temp recycling done") + pattern := filepath.Join(common.ProductTempRoot(), "*", "recycle.now") + found, err := filepath.Glob(pattern) + if err != nil { + common.Debug("Recycling failed, reason: %v", err) + return + } + for _, filename := range found { + folder := filepath.Dir(filename) + changed, err := pathlib.Modtime(folder) + if err == nil && time.Since(changed) > 48*time.Hour { + go os.RemoveAll(folder) + } + } + runtime.Gosched() +} + +func markTempForRecycling() { + if common.DisableTempManagement() { + common.Timeline("temp management disabled -- temp not marked for recycling") + return + } + if markedAlready { + return + } + target := common.ProductTempName() + if pathlib.Exists(target) { + filename := filepath.Join(target, "recycle.now") + pathlib.WriteFile(filename, []byte("True"), 0o644) + common.Debug("Marked %q for recycling.", target) + markedAlready = true + } } func main() { + defer ExitProtection() + + notify := operations.RccVersionCheck() + if notify != nil { + defer notify() + } + + warning, _ := EnsureUserRegistered() + if len(warning) > 0 { + defer pretty.Warning("%s", warning) + } + + anywork.Backlog(conda.BugsCleanup) + + if common.SharedHolotree { + common.TimelineBegin("Start [shared mode]. (parent/pid: %d/%d)", os.Getppid(), os.Getpid()) + } else { + common.TimelineBegin("Start [private mode]. (parent/pid: %d/%d)", os.Getppid(), os.Getpid()) + } + defer common.EndOfTimeline() + if common.OneOutOf(6) { + go startTempRecycling() + } + defer markTempForRecycling() defer os.Stderr.Sync() defer os.Stdout.Sync() - defer ExitProtection() cmd.Execute() + common.Timeline("Command execution done.") + if common.OneOutOf(5) { + TimezoneMetric() + } + + if common.WarrantyVoided() { + common.Timeline("Running in 'warranty voided' mode.") + pretty.Warning("Note that 'rcc' is running in 'warranty voided' mode.") + } + + anywork.Sync() } diff --git a/cmd/rccremote/main.go b/cmd/rccremote/main.go new file mode 100644 index 00000000..72f9a67b --- /dev/null +++ b/cmd/rccremote/main.go @@ -0,0 +1,79 @@ +package main + +import ( + "flag" + "os" + "path/filepath" + + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/pathlib" + "github.com/robocorp/rcc/pretty" + "github.com/robocorp/rcc/remotree" +) + +var ( + domainId string + serverName string + serverPort int + versionFlag bool + holdingArea string + debugFlag bool + traceFlag bool +) + +func defaultHoldLocation() string { + where, err := pathlib.Abs(filepath.Join(pathlib.TempDir(), "rccremotehold")) + if err != nil { + return "temphold" + } + return where +} + +func init() { + flag.BoolVar(&debugFlag, "debug", false, "Turn on debugging output.") + flag.BoolVar(&traceFlag, "trace", false, "Turn on tracing output.") + + flag.BoolVar(&versionFlag, "version", false, "Just show rccremote version and exit.") + flag.StringVar(&serverName, "hostname", "localhost", "Hostname/address to bind server to.") + flag.IntVar(&serverPort, "port", 4653, "Port to bind server in given hostname.") + flag.StringVar(&holdingArea, "hold", defaultHoldLocation(), "Directory where to put HOLD files once known.") + flag.StringVar(&domainId, "domain", "personal", "Symbolic domain that this peer serves.") +} + +func ExitProtection() { + status := recover() + if status != nil { + exit, ok := status.(common.ExitCode) + if ok { + exit.ShowMessage() + common.WaitLogs() + os.Exit(exit.Code) + } + common.WaitLogs() + panic(status) + } + common.WaitLogs() +} + +func showVersion() { + common.Stdout("%s\n", common.Version) + os.Exit(0) +} + +func process() { + if versionFlag { + showVersion() + } + pretty.Guard(common.SharedHolotree, 1, "Shared holotree must be enabled and in use for rccremote to work.") + common.Log("Remote for rcc starting (%s) ...", common.Version) + remotree.Serve(serverName, serverPort, domainId, holdingArea) +} + +func main() { + defer ExitProtection() + pretty.Setup() + + flag.Parse() + common.DefineVerbosity(false, debugFlag, traceFlag) + process() +} diff --git a/cmd/rccremote/main_test.go b/cmd/rccremote/main_test.go new file mode 100644 index 00000000..06ab7d0f --- /dev/null +++ b/cmd/rccremote/main_test.go @@ -0,0 +1 @@ +package main diff --git a/cmd/robot.go b/cmd/robot.go index b6416187..feef996d 100644 --- a/cmd/robot.go +++ b/cmd/robot.go @@ -1,6 +1,9 @@ package cmd import ( + "fmt" + + "github.com/robocorp/rcc/common" "github.com/spf13/cobra" ) @@ -8,8 +11,8 @@ var robotCmd = &cobra.Command{ Use: "robot", Aliases: []string{"r"}, Short: "Group of commands related to `robot`.", - Long: `This set of commands relate to Robocorp Cloud related tasks. They are -executed either locally, or in connection to Robocorp Cloud and Robocorp App.`, + Long: fmt.Sprintf(`This set of commands relate to %s Control Room related tasks. They are +executed either locally, or in connection to %s Control Room and tooling.`, common.Product.Name(), common.Product.Name()), } func init() { diff --git a/cmd/robotdependencies.go b/cmd/robotdependencies.go new file mode 100644 index 00000000..762a1db7 --- /dev/null +++ b/cmd/robotdependencies.go @@ -0,0 +1,63 @@ +package cmd + +import ( + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/conda" + "github.com/robocorp/rcc/operations" + "github.com/robocorp/rcc/pathlib" + "github.com/robocorp/rcc/pretty" + "github.com/robocorp/rcc/robot" + + "github.com/spf13/cobra" +) + +var ( + exportDependenciesFlag bool +) + +func doShowDependencies(config robot.Robot, label string) { + filename, _ := config.DependenciesFile() + err := conda.SideBySideViewOfDependencies(conda.GoldenMasterFilename(label), filename) + pretty.Guard(err == nil, 3, "Failed to show dependencies, reason: %v", err) +} + +func doCopyDependencies(config robot.Robot, label string) { + mode := "[create]" + target, found := config.DependenciesFile() + if found { + mode = "[overwrite]" + } + source := conda.GoldenMasterFilename(label) + common.Log("%sCopying %q as wanted %q %s.%s", pretty.Yellow, source, target, mode, pretty.Reset) + err := pathlib.CopyFile(source, target, found) + pretty.Guard(err == nil, 2, "Copy %q -> %q failed, reason: %v", source, target, err) +} + +var robotDependenciesCmd = &cobra.Command{ + Use: "dependencies", + Short: "View wanted vs. available dependencies of robot execution environment.", + Long: "View wanted vs. available dependencies of robot execution environment.", + Aliases: []string{"deps"}, + Run: func(cmd *cobra.Command, args []string) { + if common.DebugFlag() { + defer common.Stopwatch("Robot dependencies run lasted").Report() + } + simple, config, _, label := operations.LoadAnyTaskEnvironment(robotFile, forceFlag) + pretty.Guard(!simple, 1, "Cannot view dependencies of simple robots.") + if exportDependenciesFlag { + common.Log("--") + doCopyDependencies(config, label) + } + common.Log("--") + doShowDependencies(config, label) + pretty.Ok() + }, +} + +func init() { + robotCmd.AddCommand(robotDependenciesCmd) + robotDependenciesCmd.Flags().BoolVarP(&exportDependenciesFlag, "export", "e", false, "Export execution environment description into robot dependencies.yaml, overwriting previous if exists.") + robotDependenciesCmd.Flags().BoolVarP(&forceFlag, "force", "f", false, "Forced environment update.") + robotDependenciesCmd.Flags().StringVarP(&robotFile, "robot", "r", "robot.yaml", "Full path to the 'robot.yaml' configuration file.") + robotDependenciesCmd.Flags().StringVarP(&common.HolotreeSpace, "space", "s", "user", "Space to use for execution environment dependencies.") +} diff --git a/cmd/robotdiagnostics.go b/cmd/robotdiagnostics.go new file mode 100644 index 00000000..eea156a8 --- /dev/null +++ b/cmd/robotdiagnostics.go @@ -0,0 +1,32 @@ +package cmd + +import ( + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/operations" + "github.com/robocorp/rcc/pretty" + + "github.com/spf13/cobra" +) + +var robotDiagnosticsCmd = &cobra.Command{ + Use: "diagnostics", + Short: "Run system diagnostics to help resolve rcc issues.", + Long: "Run system diagnostics to help resolve rcc issues.", + Run: func(cmd *cobra.Command, args []string) { + if common.DebugFlag() { + defer common.Stopwatch("Diagnostic run lasted").Report() + } + err := operations.PrintRobotDiagnostics(robotFile, jsonFlag, productionFlag) + if err != nil { + pretty.Exit(1, "Error: %v", err) + } + pretty.Ok() + }, +} + +func init() { + robotCmd.AddCommand(robotDiagnosticsCmd) + robotDiagnosticsCmd.Flags().BoolVarP(&jsonFlag, "json", "j", false, "Output in JSON format") + robotDiagnosticsCmd.Flags().BoolVarP(&productionFlag, "production", "p", false, "Checks for production level robots.") + robotDiagnosticsCmd.Flags().StringVarP(&robotFile, "robot", "r", "robot.yaml", "Full path to the 'robot.yaml' configuration file.") +} diff --git a/cmd/robotlist.go b/cmd/robotlist.go deleted file mode 100644 index 39019a70..00000000 --- a/cmd/robotlist.go +++ /dev/null @@ -1,88 +0,0 @@ -package cmd - -import ( - "encoding/json" - "os" - "time" - - "github.com/robocorp/rcc/common" - "github.com/robocorp/rcc/operations" - "github.com/robocorp/rcc/pretty" - - "github.com/spf13/cobra" -) - -var ( - taskDirectory string -) - -const ( - timeFormat = `02.01.2006 15:04` -) - -func updateRobotDirectory(directory string) { - err := operations.UpdateRobot(directory) - if err != nil { - pretty.Exit(1, "Error: %v", err) - } -} - -func jsonRobots() { - robots, err := operations.ListRobots() - if err != nil { - pretty.Exit(1, "Error: %v", err) - } - encoder := json.NewEncoder(os.Stdout) - encoder.SetIndent("", " ") - err = encoder.Encode(robots) - if err != nil { - pretty.Exit(2, "Error: %v", err) - } -} - -func listRobots() { - if jsonFlag { - jsonRobots() - return - } - robots, err := operations.ListRobots() - if err != nil { - pretty.Exit(1, "Error: %v", err) - } - if len(robots) == 0 { - pretty.Exit(2, "Error: No robots found!") - } - common.Log("Updated at | Created at | Directory") - for _, robot := range robots { - updated := time.Unix(robot.Updated, 0) - created := time.Unix(robot.Created, 0) - status := "" - if robot.Deleted > 0 { - status = "" - } - common.Log("%v | %v | %v %v", updated.Format(timeFormat), created.Format(timeFormat), robot.Path, status) - } -} - -var robotlistCmd = &cobra.Command{ - Use: "list", - Short: "List or update tracked robot directories.", - Long: "List or update tracked robot directories.", - Args: cobra.NoArgs, - Run: func(cmd *cobra.Command, args []string) { - if common.DebugFlag { - defer common.Stopwatch("Robot list lasted").Report() - } - if len(taskDirectory) > 0 { - updateRobotDirectory(taskDirectory) - } else { - listRobots() - } - }, -} - -func init() { - robotCmd.AddCommand(robotlistCmd) - robotlistCmd.Flags().StringVarP(&taskDirectory, "add", "a", "", "The root directory to add as robot.") - robotlistCmd.Flags().BoolVarP(&jsonFlag, "json", "j", false, "Output in JSON format.") -} diff --git a/cmd/root.go b/cmd/root.go index 299153de..02895555 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -4,20 +4,35 @@ import ( "fmt" "os" "path/filepath" + "runtime/pprof" "strings" + "github.com/robocorp/rcc/anywork" "github.com/robocorp/rcc/common" "github.com/robocorp/rcc/conda" - "github.com/robocorp/rcc/operations" "github.com/robocorp/rcc/pathlib" "github.com/robocorp/rcc/pretty" + "github.com/robocorp/rcc/set" "github.com/robocorp/rcc/xviper" "github.com/spf13/cobra" ) +var ( + anythingIgnore string + profilefile string + profiling *os.File + versionFlag bool + silentFlag bool + debugFlag bool + traceFlag bool + productFakeFlag bool // this is handled in common init + + excludedCommands = []string{"completion"} +) + func toplevelCommands(parent *cobra.Command) { - common.Log("\nToplevel commands") + common.Log("\nToplevel commands (%v)", common.Version) for _, child := range parent.Commands() { if child.Hidden || len(child.Commands()) > 0 { continue @@ -34,10 +49,13 @@ func commandTree(level int, prefix string, parent *cobra.Command) { if level == 1 && len(parent.Commands()) == 0 { return } + name := strings.Split(parent.Use, " ") + if set.Member(excludedCommands, name[0]) { + return + } if level == 1 { common.Log("%s", strings.TrimSpace(prefix)) } - name := strings.Split(parent.Use, " ") label := fmt.Sprintf("%s%s", prefix, name[0]) common.Log("%-16s %s", label, parent.Short) indent := prefix + "| " @@ -48,13 +66,17 @@ func commandTree(level int, prefix string, parent *cobra.Command) { var rootCmd = &cobra.Command{ Use: "rcc", - Short: "rcc is environment manager for Robocorp Automation Stack", - Long: `rcc provides support for creating and managing tasks, -communicating with Robocorp Cloud, and managing virtual environments where -tasks can be developed, debugged, and run.`, + Short: fmt.Sprintf("rcc is environment manager for %s Automation Stack", common.Product.Name()), + Long: fmt.Sprintf(`rcc provides support for creating and managing tasks, +communicating with %s Control Room, and managing virtual environments where +tasks can be developed, debugged, and run.`, common.Product.Name()), Run: func(cmd *cobra.Command, args []string) { - commandTree(0, "", cmd.Root()) - toplevelCommands(cmd.Root()) + if versionFlag { + common.Stdout("%s\n", common.Version) + } else { + commandTree(0, "", cmd.Root()) + toplevelCommands(cmd.Root()) + } }, } @@ -69,38 +91,84 @@ func Origin() string { } func Execute() { - if err := rootCmd.Execute(); err != nil { - pretty.Exit(1, "Error: [rcc %v] %v", common.Version, err) - } + defer func() { + if profiling != nil { + common.Timeline("closing profiling started") + pprof.StopCPUProfile() + profiling.Sync() + profiling.Close() + common.TimelineEnd() + } + }() + + rootCmd.SetArgs(os.Args[1:]) + + err := rootCmd.Execute() + pretty.Guard(err == nil, 1, "Error: [rcc %v] %v", common.Version, err) } func init() { cobra.OnInitialize(initConfig) - rootCmd.PersistentFlags().StringVar(&controllerType, "controller", "user", "internal, DO NOT USE (unless you know what you are doing)") - rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $ROBOCORP/rcc.yaml)") + rootCmd.Flags().BoolVarP(&versionFlag, "version", "v", false, "Show rcc version and exit.") - rootCmd.PersistentFlags().BoolVarP(&common.Silent, "silent", "", false, "be less verbose on output") + rootCmd.PersistentFlags().StringVar(&profilefile, "pprof", "", "Filename to save profiling information.") + rootCmd.PersistentFlags().StringVar(&common.ControllerType, "controller", "user", "internal, DO NOT USE (unless you know what you are doing)") + rootCmd.PersistentFlags().StringVar(&common.SemanticTag, "tag", "transient", "semantic reason/context, why are you invoking rcc") + rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", fmt.Sprintf("config file (default is $%s/rcc.yaml)", common.Product.HomeVariable())) + rootCmd.PersistentFlags().StringVar(&anythingIgnore, "anything", "", "freeform string value that can be set without any effect, for example CLI versioning/reference") + + rootCmd.PersistentFlags().BoolVarP(&productFakeFlag, "sema4ai", "", false, "Select Sema4.ai toolset strategy.") + rootCmd.PersistentFlags().BoolVarP(&productFakeFlag, "robocorp", "", false, "Select Robocorp toolset strategy.") + rootCmd.PersistentFlags().BoolVarP(&common.NoBuild, "no-build", "", false, "never allow building new environments, only use what exists already in hololib (also RCC_NO_BUILD=1)") + rootCmd.PersistentFlags().BoolVarP(&common.NoRetryBuild, "no-retry-build", "", false, "no retry in case of first environment build fails, just report error immediately") + rootCmd.PersistentFlags().BoolVarP(&silentFlag, "silent", "", false, "be less verbose on output (also RCC_VERBOSITY=silent)") + rootCmd.PersistentFlags().BoolVarP(&common.Liveonly, "liveonly", "", false, "do not create base environment from live ... DANGER! For containers only!") rootCmd.PersistentFlags().BoolVarP(&pathlib.Lockless, "lockless", "", false, "do not use file locking ... DANGER!") + rootCmd.PersistentFlags().BoolVarP(&pretty.Colorless, "colorless", "", false, "do not use colors in CLI UI") rootCmd.PersistentFlags().BoolVarP(&common.NoCache, "nocache", "", false, "do not use cache for credentials and tokens, always request them from cloud") - rootCmd.PersistentFlags().BoolVarP(&common.DebugFlag, "debug", "", false, "to get debug output where available (not for production use)") - rootCmd.PersistentFlags().BoolVarP(&common.TraceFlag, "trace", "", false, "to get trace output where available (not for production use)") + rootCmd.PersistentFlags().BoolVarP(&common.LogLinenumbers, "numbers", "", false, "put line numbers on rcc produced log output") + rootCmd.PersistentFlags().BoolVarP(&debugFlag, "debug", "", false, "to get debug output where available (not for normal production use; also RCC_VERBOSITY=debug)") + rootCmd.PersistentFlags().BoolVarP(&traceFlag, "trace", "", false, "to get trace output where available (not for normal production use; also RCC_VERBOSITY=trace)") + rootCmd.PersistentFlags().BoolVarP(&common.TimelineEnabled, "timeline", "", false, "print timeline at the end of run") + rootCmd.PersistentFlags().BoolVarP(&common.StrictFlag, "strict", "", false, "be more strict on environment creation and handling") + rootCmd.PersistentFlags().IntVarP(&anywork.WorkerCount, "workers", "", 0, "scale background workers manually (do not use, unless you know what you are doing)") + rootCmd.PersistentFlags().BoolVarP(&common.UnmanagedSpace, "unmanaged", "", false, "work with unmanaged holotree spaces, DO NOT USE (unless you know what you are doing)") + rootCmd.PersistentFlags().BoolVarP(&common.WarrantyVoidedFlag, "warranty-voided", "", common.WarrantyVoidedFlag, "experimental, warranty voided, dangerous mode ... DO NOT USE (unless you know what you are doing)") + rootCmd.PersistentFlags().BoolVarP(&common.NoTempManagement, "no-temp-management", "", common.NoTempManagement, "rcc wont do any temp directory management ... DO NOT USE (unless you know what you are doing)") + rootCmd.PersistentFlags().BoolVarP(&common.NoPycManagement, "no-pyc-management", "", common.NoPycManagement, "rcc wont do any .pyc file management ... DO NOT USE (unless you know what you are doing)") + rootCmd.PersistentFlags().StringArrayVarP(&common.LogHides, "log-hide", "", []string{}, "hide logging output that matches given text fragment and this option can be given multiple times") + rootCmd.PersistentFlags().BoolVarP(&common.BundledFlag, "bundled", "", common.BundledFlag, "used to tell rcc, that this is bundled use (do not use, unless you know what you are doing)") } func initConfig() { + if profilefile != "" { + common.TimelineBegin("profiling run started") + sink, err := pathlib.Create(profilefile) + pretty.Guard(err == nil, 5, "Failed to create profile file %q, reason %v.", profilefile, err) + err = pprof.StartCPUProfile(sink) + pretty.Guard(err == nil, 6, "Failed to start CPU profile, reason %v.", err) + profiling = sink + } if cfgFile != "" { xviper.SetConfigFile(cfgFile) } else { - xviper.SetConfigFile(filepath.Join(conda.RobocorpHome(), "rcc.yaml")) + xviper.SetConfigFile(filepath.Join(common.Product.Home(), "rcc.yaml")) } - common.UnifyVerbosityFlags() - if len(controllerType) > 0 { - operations.BackgroundMetric("rcc", "rcc.controlled.by", controllerType) - } + common.DefineVerbosity(silentFlag, debugFlag, traceFlag) + common.UnifyStageHandling() pretty.Setup() + + if common.WarrantyVoided() { + pretty.Warning("Note that 'rcc' is running in 'warranty voided' mode.") + } + + common.Timeline("%q", os.Args) common.Trace("CLI command was: %#v", os.Args) common.Debug("Using config file: %v", xviper.ConfigFileUsed()) + conda.ValidateLocations() + anywork.AutoScale() } diff --git a/cmd/run.go b/cmd/run.go index ca5ff35a..2c75e868 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -1,18 +1,19 @@ package cmd import ( + "github.com/robocorp/rcc/cloud" "github.com/robocorp/rcc/common" "github.com/robocorp/rcc/conda" + "github.com/robocorp/rcc/journal" "github.com/robocorp/rcc/operations" - "github.com/robocorp/rcc/pretty" - "github.com/robocorp/rcc/xviper" "github.com/spf13/cobra" ) var ( - rcHosts = []string{"RC_API_SECRET_HOST", "RC_API_WORKITEM_HOST"} - rcTokens = []string{"RC_API_SECRET_TOKEN", "RC_API_WORKITEM_TOKEN"} + rcHosts = []string{"RC_API_SECRET_HOST", "RC_API_WORKITEM_HOST"} + rcTokens = []string{"RC_API_SECRET_TOKEN", "RC_API_WORKITEM_TOKEN"} + interactiveFlag bool ) var runCmd = &cobra.Command{ @@ -21,41 +22,49 @@ var runCmd = &cobra.Command{ Short: "Run task in place, to debug current setup.", Long: `Local task run, in place, to see how full run execution works in your own machine.`, - Args: cobra.NoArgs, Run: func(cmd *cobra.Command, args []string) { - if common.DebugFlag { + defer conda.RemoveCurrentTemp() + defer journal.BuildEventStats("robot") + defer journal.StopRunJournal() + if common.DebugFlag() { defer common.Stopwatch("Task run lasted").Report() } - ok := conda.MustConda() - if !ok { - pretty.Exit(2, "Could not get miniconda installed.") - } - defer xviper.RunMinutes().Done() simple, config, todo, label := operations.LoadTaskWithEnvironment(robotFile, runTask, forceFlag) - operations.BackgroundMetric("rcc", "rcc.cli.run", common.Version) - operations.SelectExecutionModel(captureRunFlags(), simple, todo.Commandline(), config, todo, label, false, nil) + cloud.InternalBackgroundMetric(common.ControllerIdentity(), "rcc.cli.run", common.Version) + commandline := todo.Commandline() + commandline = append(commandline, args...) + operations.SelectExecutionModel(captureRunFlags(false), simple, commandline, config, todo, label, interactiveFlag, nil) }, } -func captureRunFlags() *operations.RunFlags { +func captureRunFlags(assistant bool) *operations.RunFlags { return &operations.RunFlags{ + TokenPeriod: &operations.TokenPeriod{ + ValidityTime: validityTime, + GracePeriod: gracePeriod, + }, AccountName: AccountName(), WorkspaceId: workspaceId, - ValidityTime: validityTime, EnvironmentFile: environmentFile, RobotYaml: robotFile, + Assistant: assistant, } } func init() { - taskCmd.AddCommand(runCmd) rootCmd.AddCommand(runCmd) + taskCmd.AddCommand(runCmd) runCmd.Flags().StringVarP(&environmentFile, "environment", "e", "", "Full path to the 'env.json' development environment data file.") - runCmd.Flags().StringVarP(&robotFile, "robot", "r", "robot.yaml", "Full path to the 'robot.yaml' configuration file. (Backward compatibility with 'package.yaml')") + runCmd.Flags().StringVarP(&robotFile, "robot", "r", "robot.yaml", "Full path to the 'robot.yaml' configuration file.") runCmd.Flags().StringVarP(&runTask, "task", "t", "", "Task to run from the configuration file.") runCmd.Flags().StringVarP(&workspaceId, "workspace", "w", "", "Optional workspace id to get authorization tokens for. OPTIONAL") - runCmd.Flags().IntVarP(&validityTime, "minutes", "m", 0, "How many minutes the authorization should be valid for. OPTIONAL") + runCmd.Flags().IntVarP(&validityTime, "minutes", "m", 15, "How many minutes the authorization should be valid for (minimum 15 minutes).") + runCmd.Flags().IntVarP(&gracePeriod, "graceperiod", "", 5, "What is grace period buffer in minutes on top of validity minutes (minimum 5 minutes).") runCmd.Flags().StringVarP(&accountName, "account", "", "", "Account used for workspace. OPTIONAL") runCmd.Flags().BoolVarP(&forceFlag, "force", "f", false, "Force conda cache update (only for new environments).") + runCmd.Flags().BoolVarP(&interactiveFlag, "interactive", "", false, "Allow robot to be interactive in terminal/command prompt. For development only, not for production!") + runCmd.Flags().StringVarP(&common.HolotreeSpace, "space", "s", "user", "Client specific name to identify this environment.") + runCmd.Flags().BoolVarP(&common.NoOutputCapture, "no-outputs", "", false, "Do not capture stderr/stdout into files.") + runCmd.Flags().BoolVarP(&common.DeveloperFlag, "dev", "", false, "Use devTasks instead of normal tasks. For development work only. Stragegy selection.") } diff --git a/cmd/script.go b/cmd/script.go new file mode 100644 index 00000000..1d768373 --- /dev/null +++ b/cmd/script.go @@ -0,0 +1,51 @@ +package cmd + +import ( + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/operations" + + "github.com/spf13/cobra" +) + +var scriptCmd = &cobra.Command{ + Use: "script", + Short: "Run script inside robot task environment.", + Long: "Run script inside robot task environment.", + Example: ` + rcc task script -- pip list + rcc task script --silent -- python --version +`, + Args: cobra.MinimumNArgs(1), + Run: func(cmd *cobra.Command, args []string) { + if common.DebugFlag() { + defer common.Stopwatch("Task run lasted").Report() + } + simple, config, todo, label := operations.LoadAnyTaskEnvironment(robotFile, forceFlag) + operations.SelectExecutionModel(noRunFlags(), simple, args, config, todo, label, interactiveFlag, nil) + }, +} + +func noRunFlags() *operations.RunFlags { + return &operations.RunFlags{ + TokenPeriod: &operations.TokenPeriod{ + ValidityTime: 0, + GracePeriod: 0, + }, + AccountName: "", + WorkspaceId: "", + EnvironmentFile: environmentFile, + RobotYaml: robotFile, + Assistant: false, + NoPipFreeze: true, + } +} + +func init() { + taskCmd.AddCommand(scriptCmd) + + scriptCmd.Flags().StringVarP(&environmentFile, "environment", "e", "", "Full path to the 'env.json' development environment data file.") + scriptCmd.Flags().StringVarP(&robotFile, "robot", "r", "robot.yaml", "Full path to the 'robot.yaml' configuration file.") + scriptCmd.Flags().BoolVarP(&forceFlag, "force", "f", false, "Force conda cache update (only for new environments).") + scriptCmd.Flags().BoolVarP(&interactiveFlag, "interactive", "", false, "Allow robot to be interactive in terminal/command prompt. For development only, not for production!") + scriptCmd.Flags().StringVarP(&common.HolotreeSpace, "space", "s", "user", "Client specific name to identify this environment.") +} diff --git a/cmd/settings.go b/cmd/settings.go new file mode 100644 index 00000000..4cfe2950 --- /dev/null +++ b/cmd/settings.go @@ -0,0 +1,46 @@ +package cmd + +import ( + "fmt" + "os" + + "github.com/robocorp/rcc/pretty" + "github.com/robocorp/rcc/settings" + "github.com/spf13/cobra" +) + +var ( + settingsDefaults bool +) + +var settingsCmd = &cobra.Command{ + Use: "settings", + Short: "Show effective settings.yaml content.", + Long: `Show effective/active settings.yaml content. If you need DEFAULT status, use --defaults option.`, + Run: func(cmd *cobra.Command, args []string) { + switch { + case settingsDefaults: + raw, err := settings.DefaultSettings() + pretty.Guard(err == nil, 1, "Error while loading defaults: %v", err) + fmt.Fprintf(os.Stdout, "%s", string(raw)) + case jsonFlag: + config, err := settings.SummonSettings() + pretty.Guard(err == nil, 2, "Error while loading settings: %v", err) + json, err := config.AsJson() + pretty.Guard(err == nil, 3, "Error while converting settings: %v", err) + fmt.Fprintf(os.Stdout, "%s", string(json)) + default: + config, err := settings.SummonSettings() + pretty.Guard(err == nil, 2, "Error while loading settings: %v", err) + yaml, err := config.AsYaml() + pretty.Guard(err == nil, 3, "Error while converting settings: %v", err) + fmt.Fprintf(os.Stdout, "%s", string(yaml)) + } + }, +} + +func init() { + configureCmd.AddCommand(settingsCmd) + settingsCmd.Flags().BoolVarP(&settingsDefaults, "defaults", "d", false, "Show DEFAULT settings. Can be used as configuration template.") + settingsCmd.Flags().BoolVarP(&jsonFlag, "json", "j", false, "Show EFFECTIVE settings as JSON stream. For applications to use.") +} diff --git a/cmd/sharedvariables.go b/cmd/sharedvariables.go index b643db1c..34cd0914 100644 --- a/cmd/sharedvariables.go +++ b/cmd/sharedvariables.go @@ -2,12 +2,14 @@ package cmd // flags var ( - autoInstall bool - defaultFlag bool - forceFlag bool - listFlag bool - jsonFlag bool - verifiedFlag bool + autoInstall bool + defaultFlag bool + forceFlag bool + listFlag bool + jsonFlag bool + dryFlag bool + productionFlag bool + verifiedFlag bool ) // options @@ -16,7 +18,6 @@ var ( assistantId string bearerToken string cfgFile string - controllerType string copyDirectory string directory string endpointUrl string @@ -31,7 +32,9 @@ var ( runTask string shellDirectory string templateName string + gracePeriod int validityTime int workspaceId string + wskey string zipfile string ) diff --git a/cmd/shell.go b/cmd/shell.go index 362849a6..e6304fb1 100644 --- a/cmd/shell.go +++ b/cmd/shell.go @@ -18,18 +18,14 @@ It can be used to get inside a managed environment and execute your own command within that environment.`, Args: cobra.NoArgs, Run: func(cmd *cobra.Command, args []string) { - if common.DebugFlag { + if common.DebugFlag() { defer common.Stopwatch("rcc shell lasted").Report() } - ok := conda.MustConda() - if !ok { - pretty.Exit(2, "Could not get miniconda installed.") - } - simple, config, todo, label := operations.LoadTaskWithEnvironment(robotFile, runTask, forceFlag) + simple, config, todo, label := operations.LoadAnyTaskEnvironment(robotFile, forceFlag) if simple { pretty.Exit(1, "Cannot do shell for simple execution model.") } - operations.ExecuteTask(captureRunFlags(), conda.Shell, config, todo, label, true, nil) + operations.ExecuteTask(captureRunFlags(false), conda.Shell, config, todo, label, true, nil) }, } @@ -37,7 +33,8 @@ func init() { taskCmd.AddCommand(shellCmd) shellCmd.Flags().StringVarP(&environmentFile, "environment", "e", "", "Full path to the 'env.json' development environment data file.") - shellCmd.Flags().StringVarP(&robotFile, "robot", "r", "robot.yaml", "Full path to the 'robot.yaml' configuration file. (With backward compatibility with 'package.yaml')") - shellCmd.Flags().StringVarP(&runTask, "task", "t", "", "Task to configure shell from configuration file.") + shellCmd.Flags().StringVarP(&robotFile, "robot", "r", "robot.yaml", "Full path to the 'robot.yaml' configuration file.") + shellCmd.Flags().StringVarP(&common.HolotreeSpace, "space", "s", "user", "Client specific name to identify used environment.") + shellCmd.Flags().StringVarP(&runTask, "task", "t", "", "Task to configure shell from configuration file. ") shellCmd.MarkFlagRequired("config") } diff --git a/cmd/speed.go b/cmd/speed.go new file mode 100644 index 00000000..3dd5901a --- /dev/null +++ b/cmd/speed.go @@ -0,0 +1,98 @@ +package cmd + +import ( + "fmt" + "math/rand" + "os" + "path/filepath" + "runtime" + "time" + + "github.com/robocorp/rcc/anywork" + "github.com/robocorp/rcc/blobs" + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/htfs" + "github.com/robocorp/rcc/operations" + "github.com/robocorp/rcc/pathlib" + "github.com/robocorp/rcc/pretty" + "github.com/robocorp/rcc/settings" + + "github.com/spf13/cobra" +) + +var ( + randomIdentifier string +) + +func init() { + randomIdentifier = fmt.Sprintf("%016x", rand.Uint64()^uint64(os.Getpid())) +} + +func workingWorm(pipe chan bool, reply chan int, debug bool) { + if !debug { + fmt.Fprintf(os.Stderr, "\nWorking: -----") + } + seconds := 0 +loop: + for { + if !debug { + fmt.Fprintf(os.Stderr, "\b\b\b\b\b%4ds", seconds) + os.Stderr.Sync() + } + select { + case <-time.After(1 * time.Second): + seconds += 1 + continue + case <-pipe: + break loop + } + } + reply <- seconds +} + +var speedtestCmd = &cobra.Command{ + Use: "speedtest", + Aliases: []string{"speed"}, + Short: "Run system speed test to find how rcc performs in your system.", + Long: "Run system speed test to find how rcc performs in your system.", + Run: func(cmd *cobra.Command, args []string) { + if common.DebugFlag() { + defer common.Stopwatch("Speed test run lasted").Report() + } + common.Log("System %q running on %q.", settings.OperatingSystem(), common.Platform()) + common.Log("Running network and filesystem performance tests with %d workers on %d CPUs.", anywork.Scale(), runtime.NumCPU()) + common.Log("This may take several minutes, please be patient.") + signal := make(chan bool) + timing := make(chan int) + silent, debug, trace := common.Silent(), common.DebugFlag(), common.TraceFlag() + if !debug { + common.DefineVerbosity(true, false, false) + } + go workingWorm(signal, timing, debug) + folder := common.ProductTemp() + pretty.DebugNote("Speed test will force temporary %s to be %q while testing.", common.Product.HomeVariable(), folder) + err := os.RemoveAll(folder) + pretty.Guard(err == nil, 4, "Error: %v", err) + content, err := blobs.Asset("assets/speedtest.yaml") + pretty.Guard(err == nil, 1, "Error: %v", err) + condafile := filepath.Join(folder, "speedtest.yaml") + err = pathlib.WriteFile(condafile, content, 0o666) + pretty.Guard(err == nil, 2, "Error: %v", err) + common.Product.ForceHome(folder) + _, score, err := htfs.NewEnvironment(condafile, "", true, true, operations.PullCatalog) + common.DefineVerbosity(silent, debug, trace) + pretty.Guard(err == nil, 3, "Error: %v", err) + common.Product.ForceHome("") + err = os.RemoveAll(folder) + pretty.Guard(err == nil, 4, "Error: %v", err) + score.Done() + close(signal) + elapsed := <-timing + common.Log("%s", score.Score(anywork.Scale(), elapsed)) + pretty.Ok() + }, +} + +func init() { + configureCmd.AddCommand(speedtestCmd) +} diff --git a/cmd/task.go b/cmd/task.go index bbc6ef83..8d097f9b 100644 --- a/cmd/task.go +++ b/cmd/task.go @@ -1,6 +1,9 @@ package cmd import ( + "fmt" + + "github.com/robocorp/rcc/common" "github.com/spf13/cobra" ) @@ -8,10 +11,12 @@ var taskCmd = &cobra.Command{ Use: "task", Aliases: []string{"t"}, Short: "Group of commands related to `task`.", - Long: `This set of commands relate to Robocorp Cloud related tasks. They are -executed either locally, or in connection to Robocorp Cloud and Robocorp App.`, + Long: fmt.Sprintf(`This set of commands relate to %s Control Room related tasks. They are +executed either locally, or in connection to %s Control Room and tooling.`, common.Product.Name(), common.Product.Name()), } func init() { rootCmd.AddCommand(taskCmd) + + taskCmd.PersistentFlags().BoolVarP(&common.ExternallyManaged, "externally-managed", "", false, "mark created Python environments as EXTERNALLY-MANAGED (PEP 668)") } diff --git a/cmd/testrun.go b/cmd/testrun.go index 1085ccfd..11cb1d70 100644 --- a/cmd/testrun.go +++ b/cmd/testrun.go @@ -6,13 +6,13 @@ import ( "path/filepath" "time" + "github.com/robocorp/rcc/cloud" "github.com/robocorp/rcc/common" "github.com/robocorp/rcc/conda" "github.com/robocorp/rcc/operations" "github.com/robocorp/rcc/pathlib" "github.com/robocorp/rcc/pretty" "github.com/robocorp/rcc/robot" - "github.com/robocorp/rcc/xviper" "github.com/spf13/cobra" ) @@ -23,17 +23,12 @@ var testrunCmd = &cobra.Command{ Short: "Run a task in a clean environment and clean directory.", Long: "Run a task in a clean environment and clean directory.", Run: func(cmd *cobra.Command, args []string) { - if common.DebugFlag { + defer conda.RemoveCurrentTemp() + if common.DebugFlag() { defer common.Stopwatch("Task testrun lasted").Report() } - ok := conda.MustConda() - if !ok { - pretty.Exit(4, "Could not get miniconda installed.") - } - defer xviper.RunMinutes().Done() now := time.Now() - marker := now.Unix() - zipfile := filepath.Join(os.TempDir(), fmt.Sprintf("testrun%x.zip", marker)) + zipfile := filepath.Join(pathlib.TempDir(), fmt.Sprintf("testrun%x.zip", common.When)) defer os.Remove(zipfile) common.Debug("Using temporary zip file: %v", zipfile) sourceDir := filepath.Dir(robotFile) @@ -47,10 +42,10 @@ var testrunCmd = &cobra.Command{ pretty.Exit(2, "Error: %v", err) } sentinelTime := time.Now() - workarea := filepath.Join(os.TempDir(), fmt.Sprintf("workarea%x", marker)) + workarea := filepath.Join(pathlib.TempDir(), fmt.Sprintf("workarea%x", common.When)) defer os.RemoveAll(workarea) common.Debug("Using temporary workarea: %v", workarea) - err = operations.Unzip(workarea, zipfile, false, true) + err = operations.Unzip(workarea, zipfile, false, true, true) if err != nil { pretty.Exit(3, "Error: %v", err) } @@ -58,8 +53,10 @@ var testrunCmd = &cobra.Command{ targetRobot := robot.DetectConfigurationName(workarea) simple, config, todo, label := operations.LoadTaskWithEnvironment(targetRobot, runTask, forceFlag) defer common.Log("Moving outputs to %v directory.", testrunDir) - operations.BackgroundMetric("rcc", "rcc.cli.testrun", common.Version) - operations.SelectExecutionModel(captureRunFlags(), simple, todo.Commandline(), config, todo, label, false, nil) + cloud.InternalBackgroundMetric(common.ControllerIdentity(), "rcc.cli.testrun", common.Version) + commandline := todo.Commandline() + commandline = append(commandline, args...) + operations.SelectExecutionModel(captureRunFlags(false), simple, commandline, config, todo, label, false, nil) }, } @@ -86,10 +83,13 @@ func init() { testrunCmd.Flags().StringArrayVarP(&ignores, "ignore", "i", []string{}, "File with ignore patterns.") testrunCmd.Flags().StringVarP(&environmentFile, "environment", "e", "", "Full path to the 'env.json' development environment data file.") - testrunCmd.Flags().StringVarP(&robotFile, "robot", "r", "robot.yaml", "Full path to the 'robot.yaml' configuration file. (Backward compatibility with 'package.yaml')") + testrunCmd.Flags().StringVarP(&robotFile, "robot", "r", "robot.yaml", "Full path to the 'robot.yaml' configuration file.") testrunCmd.Flags().StringVarP(&runTask, "task", "t", "", "Task to run from configuration file.") testrunCmd.Flags().StringVarP(&workspaceId, "workspace", "w", "", "Optional workspace id to get authorization tokens for. OPTIONAL") - testrunCmd.Flags().IntVarP(&validityTime, "minutes", "m", 0, "How many minutes the authorization should be valid for. OPTIONAL") + testrunCmd.Flags().IntVarP(&validityTime, "minutes", "m", 15, "How many minutes the authorization should be valid for (minimum 15 minutes).") + testrunCmd.Flags().IntVarP(&gracePeriod, "graceperiod", "", 5, "What is grace period buffer in minutes on top of validity minutes (minimum 5 minutes).") testrunCmd.Flags().StringVarP(&accountName, "account", "", "", "Account used for workspace. OPTIONAL") testrunCmd.Flags().BoolVarP(&forceFlag, "force", "f", false, "Force conda cache update. (only for new environments)") + testrunCmd.Flags().StringVarP(&common.HolotreeSpace, "space", "s", "user", "Client specific name to identify this environment.") + testrunCmd.Flags().BoolVarP(&common.NoOutputCapture, "no-outputs", "", false, "Do not capture stderr/stdout into files.") } diff --git a/cmd/tutorial.go b/cmd/tutorial.go deleted file mode 100644 index 40371f4d..00000000 --- a/cmd/tutorial.go +++ /dev/null @@ -1,28 +0,0 @@ -package cmd - -import ( - "github.com/robocorp/rcc/blobs" - "github.com/robocorp/rcc/common" - "github.com/robocorp/rcc/pretty" - - "github.com/spf13/cobra" -) - -var tutorialCmd = &cobra.Command{ - Use: "tutorial", - Short: "Show the rcc tutorial.", - Long: "Show the rcc tutorial.", - Aliases: []string{"tut"}, - Run: func(cmd *cobra.Command, args []string) { - content, err := blobs.Asset("assets/man/tutorial.txt") - if err != nil { - pretty.Exit(1, "Cannot show tutorial text, reason: %v", err) - } - common.Stdout("%s\n", content) - }, -} - -func init() { - manCmd.AddCommand(tutorialCmd) - rootCmd.AddCommand(tutorialCmd) -} diff --git a/cmd/unwrap.go b/cmd/unwrap.go index 7fe396c8..5c747956 100644 --- a/cmd/unwrap.go +++ b/cmd/unwrap.go @@ -15,10 +15,10 @@ var unwrapCmd = &cobra.Command{ robot filename, and target directory. And using --force option, files will be overwritten.`, Run: func(cmd *cobra.Command, args []string) { - if common.DebugFlag { + if common.DebugFlag() { defer common.Stopwatch("Unwrap lasted").Report() } - err := operations.Unzip(directory, zipfile, forceFlag, false) + err := operations.Unzip(directory, zipfile, forceFlag, false, true) if err != nil { pretty.Exit(1, "Error: %v", err) } diff --git a/cmd/upload.go b/cmd/upload.go index f64b5e5f..9b361c30 100644 --- a/cmd/upload.go +++ b/cmd/upload.go @@ -1,6 +1,8 @@ package cmd import ( + "fmt" + "github.com/robocorp/rcc/cloud" "github.com/robocorp/rcc/common" "github.com/robocorp/rcc/operations" @@ -11,21 +13,21 @@ import ( var uploadCmd = &cobra.Command{ Use: "upload", - Short: "Push an existing robot to Robocorp Cloud.", - Long: "Push an existing robot to Robocorp Cloud.", + Short: fmt.Sprintf("Push an existing robot to %s Control Room.", common.Product.Name()), + Long: fmt.Sprintf("Push an existing robot to %s Control Room.", common.Product.Name()), Run: func(cmd *cobra.Command, args []string) { - if common.DebugFlag { + if common.DebugFlag() { defer common.Stopwatch("Upload lasted").Report() } account := operations.AccountByName(AccountName()) if account == nil { - pretty.Exit(1, "Could not find account by name: %v", AccountName()) + pretty.Exit(1, "Could not find account by name: %q", AccountName()) } client, err := cloud.NewClient(account.Endpoint) if err != nil { pretty.Exit(2, "Could not create client for endpoint: %v, reason: %v", account.Endpoint, err) } - err = operations.UploadCommand(client, account, workspaceId, robotId, zipfile, common.DebugFlag) + err = operations.UploadCommand(client, account, workspaceId, robotId, zipfile, common.DebugFlag()) if err != nil { pretty.Exit(3, "Error: %v", err) } diff --git a/cmd/userinfo.go b/cmd/userinfo.go index 5735a8ec..5dea96bb 100644 --- a/cmd/userinfo.go +++ b/cmd/userinfo.go @@ -2,6 +2,7 @@ package cmd import ( "encoding/json" + "fmt" "github.com/robocorp/rcc/cloud" "github.com/robocorp/rcc/common" @@ -14,15 +15,15 @@ import ( var userinfoCmd = &cobra.Command{ Use: "userinfo", Aliases: []string{"user"}, - Short: "Query user information from Robocorp Cloud.", - Long: "Query user information from Robocorp Cloud.", + Short: fmt.Sprintf("Query user information from %s Control Room.", common.Product.Name()), + Long: fmt.Sprintf("Query user information from %s Control Room.", common.Product.Name()), Run: func(cmd *cobra.Command, args []string) { - if common.DebugFlag { + if common.DebugFlag() { defer common.Stopwatch("Userinfo query lasted").Report() } account := operations.AccountByName(AccountName()) if account == nil { - pretty.Exit(1, "Error: Could not find account by name: %v", AccountName()) + pretty.Exit(1, "Error: Could not find account by name: %q", AccountName()) } client, err := cloud.NewClient(account.Endpoint) if err != nil { diff --git a/cmd/variables.go b/cmd/variables.go deleted file mode 100644 index 7839effb..00000000 --- a/cmd/variables.go +++ /dev/null @@ -1,155 +0,0 @@ -package cmd - -import ( - "errors" - "fmt" - "strings" - - "github.com/robocorp/rcc/common" - "github.com/robocorp/rcc/conda" - "github.com/robocorp/rcc/operations" - "github.com/robocorp/rcc/pretty" - "github.com/robocorp/rcc/robot" - - "github.com/spf13/cobra" -) - -func Has(value string) bool { - return len(value) > 0 -} - -func asSimpleMap(line string) map[string]string { - parts := strings.SplitN(strings.TrimSpace(line), "=", 2) - if len(parts) != 2 { - return nil - } - result := make(map[string]string) - result["key"] = parts[0] - result["value"] = parts[1] - return result -} - -func asJson(items []string) error { - result := make([]map[string]string, 0, len(items)) - for _, line := range items { - entry := asSimpleMap(line) - if entry != nil { - result = append(result, entry) - } - } - content, err := operations.NiceJsonOutput(result) - if err != nil { - return err - } - common.Stdout("%s\n", content) - return nil -} - -func asText(items []string) { - for _, line := range items { - common.Stdout("%s\n", line) - } -} - -func exportEnvironment(condaYaml []string, packfile, taskName, environment, workspace string, validity int, jsonform bool) error { - var err error - var config robot.Robot - var task robot.Task - var extra []string - var data operations.Token - - if Has(packfile) { - config, err = robot.LoadYamlConfiguration(packfile) - if err == nil { - condaYaml = append(condaYaml, config.CondaConfigFile()) - task = config.TaskByName(taskName) - } - } - - if Has(environment) { - developmentEnvironment, err := robot.LoadEnvironmentSetup(environmentFile) - if err == nil { - extra = developmentEnvironment.AsEnvironment() - } - } - - if len(condaYaml) < 1 { - return errors.New("No robot.yaml, package.yaml or conda.yaml files given. Cannot continue.") - } - - label, err := conda.NewEnvironment(forceFlag, condaYaml...) - if err != nil { - return err - } - - env := conda.EnvironmentExtensionFor(label) - if task != nil { - env = task.ExecutionEnvironment(config, label, extra, false) - } - - if Has(workspace) { - claims := operations.RunClaims(validity*60, workspace) - data, err = operations.AuthorizeClaims(AccountName(), claims) - } - - if err != nil { - return err - } - - if len(data) > 0 { - endpoint := data["endpoint"] - for _, key := range rcHosts { - env = append(env, fmt.Sprintf("%s=%s", key, endpoint)) - } - token := data["token"] - for _, key := range rcTokens { - env = append(env, fmt.Sprintf("%s=%s", key, token)) - } - env = append(env, fmt.Sprintf("RC_WORKSPACE_ID=%s", workspaceId)) - } - - if jsonform { - return asJson(env) - } - - asText(env) - return nil -} - -var variablesCmd = &cobra.Command{ - Use: "variables ", - Aliases: []string{"vars"}, - Short: "Export environment specific variables as a JSON structure.", - Long: "Export environment specific variables as a JSON structure.", - Run: func(cmd *cobra.Command, args []string) { - silent := common.Silent - common.Silent = true - - defer func() { - common.Silent = silent - }() - - ok := conda.MustConda() - if !ok { - pretty.Exit(2, "Could not get miniconda installed.") - } - err := exportEnvironment(args, robotFile, runTask, environmentFile, workspaceId, validityTime, jsonFlag) - if err != nil { - pretty.Exit(1, "Error: Variable exporting failed because: %v", err) - } - }, -} - -func init() { - envCmd.AddCommand(variablesCmd) - - variablesCmd.Flags().StringVarP(&environmentFile, "environment", "e", "", "Full path to 'env.json' development environment data file. ") - variablesCmd.Flags().StringVarP(&robotFile, "robot", "r", "robot.yaml", "Full path to 'robot.yaml' configuration file. (Backward compatibility with 'package.yaml') ") - variablesCmd.Flags().StringVarP(&runTask, "task", "t", "", "Task to run from configuration file. ") - variablesCmd.Flags().StringVarP(&workspaceId, "workspace", "w", "", "Optional workspace id to get authorization tokens for. ") - variablesCmd.Flags().IntVarP(&validityTime, "minutes", "m", 0, "How many minutes the authorization should be valid for. ") - variablesCmd.Flags().StringVarP(&accountName, "account", "", "", "Account used for workspace. ") - - variablesCmd.Flags().BoolVarP(&jsonFlag, "json", "j", false, "Output in JSON format") - variablesCmd.Flags().BoolVarP(&forceFlag, "force", "f", false, "Force conda cache update. (only for new environments)") -} diff --git a/cmd/wizardcreate.go b/cmd/wizardcreate.go index 9b9e5012..7926eb60 100644 --- a/cmd/wizardcreate.go +++ b/cmd/wizardcreate.go @@ -8,8 +8,6 @@ import ( "github.com/spf13/cobra" ) -var altFlag bool - var wizardCreateCmd = &cobra.Command{ Use: "create", Short: "Create a directory structure for a robot interactively.", @@ -18,25 +16,19 @@ var wizardCreateCmd = &cobra.Command{ if !pretty.Interactive { pretty.Exit(1, "This is for interactive use only. Do not use in scripting/CI!") } - if common.DebugFlag { + if common.DebugFlag() { defer common.Stopwatch("Interactive create lasted").Report() } - if altFlag { - err := wizard.AltCreate(args) - if err != nil { - pretty.Exit(2, "%v", err) - } - } else { - err := wizard.Create(args) - if err != nil { - pretty.Exit(2, "%v", err) - } + err := wizard.Create(args) + if err != nil { + pretty.Exit(2, "%v", err) } }, } func init() { - interactiveCmd.AddCommand(wizardCreateCmd) - rootCmd.AddCommand(wizardCreateCmd) - wizardCreateCmd.Flags().BoolVarP(&altFlag, "alt", "a", false, "select alternative create command") + if common.Product.IsLegacy() { + interactiveCmd.AddCommand(wizardCreateCmd) + rootCmd.AddCommand(wizardCreateCmd) + } } diff --git a/cmd/workspace.go b/cmd/workspace.go index 6d73ab92..64f1fab0 100644 --- a/cmd/workspace.go +++ b/cmd/workspace.go @@ -16,12 +16,12 @@ var workspaceCmd = &cobra.Command{ Short: "List the available workspaces and their tasks (with --workspace option).", Long: "List the available workspaces and their tasks (with --workspace option).", Run: func(cmd *cobra.Command, args []string) { - if common.DebugFlag { + if common.DebugFlag() { defer common.Stopwatch("Workspace query lasted").Report() } account := operations.AccountByName(AccountName()) if account == nil { - pretty.Exit(1, "Could not find account by name: %v", AccountName()) + pretty.Exit(1, "Could not find account by name: %q", AccountName()) } client, err := cloud.NewClient(account.Endpoint) if err != nil { diff --git a/cmd/wrap.go b/cmd/wrap.go index d3b735b4..b2bd352d 100644 --- a/cmd/wrap.go +++ b/cmd/wrap.go @@ -15,7 +15,7 @@ var wrapCmd = &cobra.Command{ filename, source directory and optional ignore files. When wrap is run again existing robot file will silently be overwritten..`, Run: func(cmd *cobra.Command, args []string) { - if common.DebugFlag { + if common.DebugFlag() { defer common.Stopwatch("Wrap lasted").Report() } err := operations.Zip(directory, zipfile, ignores) diff --git a/common/algorithms.go b/common/algorithms.go index 3a898c0d..bb960dad 100644 --- a/common/algorithms.go +++ b/common/algorithms.go @@ -1,7 +1,14 @@ package common import ( + "crypto/sha256" + "fmt" + "hash" "math" + "math/rand" + "time" + + "github.com/dchest/siphash" ) func Entropy(input []byte) float64 { @@ -24,3 +31,71 @@ func Entropy(input []byte) float64 { } return entropy / 8.0 } + +func Hexdigest(raw []byte) string { + return fmt.Sprintf("%02x", raw) +} + +func NewDigester(legacy bool) hash.Hash { + if legacy { + return sha256.New() + } else { + return siphash.New128([]byte("Very_Random-Seed")) + } +} + +func ShortDigest(content string) string { + digester := NewDigester(true) + digester.Write([]byte(content)) + result := Hexdigest(digester.Sum(nil)) + return result[:16] +} + +func Digest(content string) string { + digester := NewDigester(true) + digester.Write([]byte(content)) + return Hexdigest(digester.Sum(nil)) +} + +func Siphash(left, right uint64, body []byte) uint64 { + return siphash.Hash(left, right, body) +} + +func DayCountSince(timestamp time.Time) int { + duration := time.Since(timestamp) + days := math.Floor(duration.Hours() / 24.0) + return int(days) +} + +func OneOutOf(limit uint8) bool { + if limit > 1 { + return rand.Intn(int(limit)) == 0 + } + return true +} + +func BlueprintHash(blueprint []byte) string { + return Textual(Sipit(blueprint), 0) +} + +func Sipit(key []byte) uint64 { + return Siphash(9007199254740993, 2147483647, key) +} + +func Textual(key uint64, size int) string { + text := fmt.Sprintf("%016x", key) + if size > 0 { + return text[:size] + } + return text +} + +func Gcd(left, right int64) int64 { + for left != 0 { + left, right = right%left, left + } + if right == 0 { + return 1 + } + return right +} diff --git a/common/algorithms_test.go b/common/algorithms_test.go index 5972bd83..947c1cf1 100644 --- a/common/algorithms_test.go +++ b/common/algorithms_test.go @@ -23,3 +23,14 @@ func TestCanCallEntropyFunction(t *testing.T) { must_be.True(between(0.5, common.Entropy([]byte("abcdefghijklmnopqrstuvwxyz")), 0.6)) must_be.True(between(0.43, common.Entropy([]byte("edf3419283feac3d4f8bb34aa9")), 0.44)) } + +func TestCanCalculateGcd(t *testing.T) { + must_be, _ := hamlet.Specifications(t) + + must_be.Equal(int64(5), common.Gcd(15, 20)) + must_be.Equal(int64(1), common.Gcd(3, 5)) + must_be.Equal(int64(5), common.Gcd(5, 5)) + must_be.Equal(int64(5), common.Gcd(0, 5)) + must_be.Equal(int64(5), common.Gcd(5, 0)) + must_be.Equal(int64(1), common.Gcd(0, 0)) +} diff --git a/common/categories.go b/common/categories.go new file mode 100644 index 00000000..b2f540c9 --- /dev/null +++ b/common/categories.go @@ -0,0 +1,21 @@ +package common + +const ( + CategoryUndefined = 0 + CategoryLongPath = 1010 + CategoryLockFile = 1020 + CategoryLockPid = 1021 + CategoryPathCheck = 1030 + CategoryEnvVarCheck = 1040 + CategoryHolotreeShared = 2010 + CategoryProductHome = 3010 + CategoryProductHomeMembers = 3020 + CategoryNetworkDNS = 4010 + CategoryNetworkLink = 4020 + CategoryNetworkHEAD = 4030 + CategoryNetworkCanary = 4040 + CategoryNetworkTLSVersion = 4050 + CategoryNetworkTLSVerify = 4060 + CategoryNetworkTLSChain = 4070 + CategoryEnvironmentCache = 5010 +) diff --git a/common/commander.go b/common/commander.go new file mode 100644 index 00000000..a4da75ee --- /dev/null +++ b/common/commander.go @@ -0,0 +1,30 @@ +package common + +import "strings" + +type Commander struct { + command []string +} + +func (it *Commander) Option(name, value string) *Commander { + value = strings.TrimSpace(value) + if len(value) > 0 { + it.command = append(it.command, name, value) + } + return it +} + +func (it *Commander) ConditionalFlag(condition bool, details ...string) *Commander { + if condition { + it.command = append(it.command, details...) + } + return it +} + +func (it *Commander) CLI() []string { + return it.command +} + +func NewCommander(parts ...string) *Commander { + return &Commander{parts} +} diff --git a/common/diagnostics.go b/common/diagnostics.go new file mode 100644 index 00000000..58c836d5 --- /dev/null +++ b/common/diagnostics.go @@ -0,0 +1,98 @@ +package common + +import ( + "encoding/json" + "fmt" + "path/filepath" + + "github.com/robocorp/rcc/fail" +) + +const ( + StatusOk = `ok` + StatusWarning = `warning` + StatusFail = `fail` + StatusFatal = `fatal` +) + +type Diagnoser func(category uint64, status, link, form string, details ...interface{}) + +func (it Diagnoser) Ok(category uint64, form string, details ...interface{}) { + it(category, StatusOk, "", form, details...) +} + +func (it Diagnoser) Warning(category uint64, link, form string, details ...interface{}) { + it(category, StatusWarning, link, form, details...) +} + +func (it Diagnoser) Fail(category uint64, link, form string, details ...interface{}) { + it(category, StatusFail, link, form, details...) +} + +func (it Diagnoser) Fatal(category uint64, link, form string, details ...interface{}) { + it(category, StatusFatal, link, form, details...) +} + +type DiagnosticStatus struct { + Details map[string]string `json:"details"` + Checks []*DiagnosticCheck `json:"checks"` +} + +type DiagnosticCheck struct { + Type string `json:"type"` + Category uint64 `json:"category"` + Status string `json:"status"` + Message string `json:"message"` + Link string `json:"url"` +} + +func (it *DiagnosticStatus) check(category uint64, kind, status, message, link string) { + it.Checks = append(it.Checks, &DiagnosticCheck{ + Type: kind, + Category: category, + Status: status, + Message: message, + Link: link, + }) +} + +func (it *DiagnosticStatus) Diagnose(kind string) Diagnoser { + return func(category uint64, status, link, form string, details ...interface{}) { + it.check(category, kind, status, fmt.Sprintf(form, details...), link) + } +} + +func (it *DiagnosticStatus) Counts() (fatal, fail, warning, ok int) { + result := make(map[string]int) + for _, check := range it.Checks { + result[check.Status] += 1 + } + return result[StatusFatal], result[StatusFail], result[StatusWarning], result[StatusOk] +} + +func (it *DiagnosticStatus) AsJson() (string, error) { + body, err := json.MarshalIndent(it, "", " ") + if err != nil { + return "", err + } + return string(body), nil +} + +func IsInsideProductHome(location string) (_ bool, err error) { + defer fail.Around(&err) + + candidate, err := filepath.Abs(location) + fail.On(err != nil, "Failed to get absolute path to %q, reason: %v", location, err) + + rchome, err := filepath.Abs(Product.Home()) + fail.On(err != nil, "Failed to get absolute path to %s, reason: %v", Product.HomeVariable(), err) + + for len(rchome) <= len(candidate) { + if rchome == candidate { + return true, nil + } + candidate = filepath.Dir(candidate) + } + + return false, nil +} diff --git a/common/elapsed.go b/common/elapsed.go index 0e260a5c..81556809 100644 --- a/common/elapsed.go +++ b/common/elapsed.go @@ -10,6 +10,24 @@ type stopwatch struct { started time.Time } +type Duration time.Duration + +func (it Duration) Seconds() float64 { + return float64(it.Truncate(time.Millisecond)) / float64(time.Second) +} + +func (it Duration) Truncate(granularity time.Duration) Duration { + return Duration(time.Duration(it).Truncate(granularity)) +} + +func (it Duration) Milliseconds() int64 { + return time.Duration(it).Milliseconds() +} + +func (it Duration) String() string { + return fmt.Sprintf("%5.3f", float64(it.Milliseconds())/1000.0) +} + func Stopwatch(form string, details ...interface{}) *stopwatch { message := fmt.Sprintf(form, details...) return &stopwatch{ @@ -18,19 +36,45 @@ func Stopwatch(form string, details ...interface{}) *stopwatch { } } +func (it *stopwatch) When() int64 { + return it.started.Unix() +} + +func (it *stopwatch) Time() time.Time { + return it.started +} + func (it *stopwatch) String() string { - elapsed := time.Now().Sub(it.started) + elapsed := it.Elapsed().Truncate(time.Millisecond) return fmt.Sprintf("%v", elapsed) } -func (it *stopwatch) Log() time.Duration { - elapsed := time.Now().Sub(it.started) - Log("%v %v", it.message, elapsed) +func (it *stopwatch) Elapsed() Duration { + return Duration(time.Since(it.started)) +} + +func (it *stopwatch) Debug() Duration { + humane, elapsed := it.explained() + Debug(humane) return elapsed } -func (it *stopwatch) Report() time.Duration { - elapsed := time.Now().Sub(it.started) - Log("%v %v", it.message, elapsed) +func (it *stopwatch) Log() Duration { + humane, elapsed := it.explained() + Log(humane) return elapsed } + +func (it *stopwatch) Report() Duration { + return it.Log() +} + +func (it *stopwatch) Text() string { + humane, _ := it.explained() + return humane +} + +func (it *stopwatch) explained() (string, Duration) { + elapsed := it.Elapsed() + return fmt.Sprintf("%s %ss", it.message, elapsed), elapsed +} diff --git a/common/elapsed_test.go b/common/elapsed_test.go index 0b90f96c..cdc8d7eb 100644 --- a/common/elapsed_test.go +++ b/common/elapsed_test.go @@ -13,6 +13,6 @@ func TestCanUseStopwatch(t *testing.T) { sut := common.Stopwatch("hello") wont_be.Nil(sut) - limit := time.Duration(10) * time.Millisecond + limit := common.Duration(10 * time.Millisecond) must_be.True(sut.Report() < limit) } diff --git a/common/exit.go b/common/exit.go index 30e68071..bf10c439 100644 --- a/common/exit.go +++ b/common/exit.go @@ -14,8 +14,12 @@ func (it ExitCode) ShowMessage() { } func Exit(code int, format string, rest ...interface{}) { + message := format + if len(rest) > 0 { + message = fmt.Sprintf(format, rest...) + } panic(ExitCode{ Code: code, - Message: fmt.Sprintf(format, rest...), + Message: message, }) } diff --git a/common/identities.go b/common/identities.go index 1ca3b0ec..1669c434 100644 --- a/common/identities.go +++ b/common/identities.go @@ -20,6 +20,7 @@ func identityProvider(sink chan string) { func init() { Startup = time.Now() + Identities = make(chan string, 3) go identityProvider(Identities) } diff --git a/common/journal.go b/common/journal.go new file mode 100644 index 00000000..964e3832 --- /dev/null +++ b/common/journal.go @@ -0,0 +1,22 @@ +package common + +var ( + journal runJournal +) + +type ( + runJournal interface { + Post(string, string, string, ...interface{}) error + } +) + +func RegisterJournal(target runJournal) { + journal = target +} + +func RunJournal(event, detail, commentForm string, fields ...interface{}) error { + if journal != nil { + return journal.Post(event, detail, commentForm, fields...) + } + return nil +} diff --git a/common/logger.go b/common/logger.go index 420ad7c7..1b94e0f4 100644 --- a/common/logger.go +++ b/common/logger.go @@ -3,38 +3,122 @@ package common import ( "fmt" "os" + "runtime" + "strings" + "sync" + "time" ) +var ( + logsource = make(logwriters) + logbarrier = sync.WaitGroup{} +) + +type logwriter func() (*os.File, string) +type logwriters chan logwriter + +func loggerLoop(writers logwriters) { + var stamp string + line := uint64(0) + for { + line += 1 + todo, ok := <-writers + if !ok { + continue + } + out, message := todo() + + if TraceFlag() { + stamp = time.Now().Format("02.150405.000 ") + } else if LogLinenumbers { + stamp = fmt.Sprintf("%3d ", line) + } else { + stamp = "" + } + fmt.Fprintf(out, "%s%s\n", stamp, message) + out.Sync() + logbarrier.Done() + } +} + +func init() { + go loggerLoop(logsource) +} + +func AcceptableOutput(message string) bool { + for _, fragment := range LogHides { + if strings.Contains(message, fragment) { + return false + } + } + return true +} + +func printout(out *os.File, message string) { + if AcceptableOutput(message) { + logbarrier.Add(1) + logsource <- func() (*os.File, string) { + return out, message + } + } +} + +func Fatal(context string, err error) { + if err != nil { + printout(os.Stderr, fmt.Sprintf("Fatal [%s]: %v", context, err)) + } +} + func Error(context string, err error) { if err != nil { Log("Error [%s]: %v", context, err) } } +func Uncritical(context string, err error) { + if err != nil { + Log("Warning [%s; not critical]: %v", context, err) + } +} + func Log(format string, details ...interface{}) { - if !Silent { - fmt.Fprintln(os.Stderr, fmt.Sprintf(format, details...)) - os.Stderr.Sync() + if !Silent() { + prefix := "" + if DebugFlag() || TraceFlag() { + prefix = "[N] " + } + printout(os.Stderr, fmt.Sprintf(prefix+format, details...)) } } func Debug(format string, details ...interface{}) error { - if DebugFlag { - fmt.Fprintln(os.Stderr, fmt.Sprintf(format, details...)) - os.Stderr.Sync() + if DebugFlag() { + printout(os.Stderr, fmt.Sprintf("[D] "+format, details...)) } return nil } func Trace(format string, details ...interface{}) error { - if TraceFlag { - fmt.Fprintln(os.Stderr, fmt.Sprintf(format, details...)) - os.Stderr.Sync() + if TraceFlag() { + printout(os.Stderr, fmt.Sprintf("[T] "+format, details...)) } return nil } func Stdout(format string, details ...interface{}) { - fmt.Fprintf(os.Stdout, format, details...) - os.Stdout.Sync() + message := format + if len(details) > 0 { + message = fmt.Sprintf(format, details...) + } + if AcceptableOutput(message) { + fmt.Fprint(os.Stdout, message) + os.Stdout.Sync() + } +} + +func WaitLogs() { + defer Timeline("wait logs done") + + runtime.Gosched() + logbarrier.Wait() } diff --git a/common/platform_darwin.go b/common/platform_darwin.go new file mode 100644 index 00000000..c7b7dafe --- /dev/null +++ b/common/platform_darwin.go @@ -0,0 +1,38 @@ +package common + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "time" +) + +const ( + defaultRobocorpLocation = "$HOME/.robocorp" + defaultHoloLocation = "/Users/Shared/robocorp/ht" + + defaultSema4Location = "$HOME/.sema4ai" + defaultSema4HoloLocation = "/Users/Shared/sema4ai/ht" +) + +func ExpandPath(entry string) string { + intermediate := os.ExpandEnv(entry) + result, err := filepath.Abs(intermediate) + if err != nil { + return intermediate + } + return result +} + +func GenerateKillCommand(keys []int) string { + command := []string{"kill -9"} + for _, key := range keys { + command = append(command, fmt.Sprintf("%d", key)) + } + return strings.Join(command, " ") +} + +func PlatformSyncDelay() { + time.Sleep(3 * time.Millisecond) +} diff --git a/common/platform_linux.go b/common/platform_linux.go new file mode 100644 index 00000000..77527389 --- /dev/null +++ b/common/platform_linux.go @@ -0,0 +1,38 @@ +package common + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "time" +) + +const ( + defaultRobocorpLocation = "$HOME/.robocorp" + defaultHoloLocation = "/opt/robocorp/ht" + + defaultSema4Location = "$HOME/.sema4ai" + defaultSema4HoloLocation = "/opt/sema4ai/ht" +) + +func ExpandPath(entry string) string { + intermediate := os.ExpandEnv(entry) + result, err := filepath.Abs(intermediate) + if err != nil { + return intermediate + } + return result +} + +func GenerateKillCommand(keys []int) string { + command := []string{"kill -9"} + for _, key := range keys { + command = append(command, fmt.Sprintf("%d", key)) + } + return strings.Join(command, " ") +} + +func PlatformSyncDelay() { + time.Sleep(3 * time.Millisecond) +} diff --git a/common/platform_windows.go b/common/platform_windows.go new file mode 100644 index 00000000..a49322d1 --- /dev/null +++ b/common/platform_windows.go @@ -0,0 +1,56 @@ +package common + +import ( + "fmt" + "os" + "path/filepath" + "regexp" + "strings" + "time" +) + +const ( + defaultRobocorpLocation = "%LOCALAPPDATA%\\robocorp" + defaultHoloLocation = "%ProgramData%\\robocorp\\ht" + + defaultSema4Location = "%LOCALAPPDATA%\\sema4ai" + defaultSema4HoloLocation = "%ProgramData%\\sema4ai\\ht" +) + +var ( + variablePattern = regexp.MustCompile("%[a-zA-Z]+%") +) + +func ExpandPath(entry string) string { + intermediate := os.ExpandEnv(entry) + intermediate = variablePattern.ReplaceAllStringFunc(intermediate, fromEnvironment) + result, err := filepath.Abs(intermediate) + if err != nil { + return intermediate + } + return result +} + +func fromEnvironment(form string) string { + replacement, ok := os.LookupEnv(form[1 : len(form)-1]) + if ok { + return replacement + } + replacement, ok = os.LookupEnv(form) + if ok { + return replacement + } + return form +} + +func GenerateKillCommand(keys []int) string { + command := []string{"taskkill /f"} + for _, key := range keys { + command = append(command, fmt.Sprintf("/pid %d", key)) + } + return strings.Join(command, " ") +} + +func PlatformSyncDelay() { + time.Sleep(300 * time.Millisecond) +} diff --git a/common/scorecard.go b/common/scorecard.go new file mode 100644 index 00000000..17acb337 --- /dev/null +++ b/common/scorecard.go @@ -0,0 +1,66 @@ +package common + +import ( + "fmt" + "time" +) + +const ( + perfMessage = ` + +Here are performance score results. Higher is better, 0 is reference point. + +Score for network=%d, filesystem=%d, and time=%d with %d workers on %q. +` + topScale = 125 + netScale = 1000 + fsScale = 100 +) + +type Scorecard interface { + Start() Scorecard + Midpoint() Scorecard + Done() Scorecard + Score(uint64, int) string +} + +type scorecard struct { + start time.Time + network time.Time + filesystem time.Time +} + +func (it *scorecard) Score(scale uint64, seconds int) string { + network := it.network.Sub(it.start).Milliseconds() + filesystem := it.filesystem.Sub(it.network).Milliseconds() + Debug("Raw score values: network=%d and filesystem=%d", network, filesystem) + if network < 1 || filesystem < 0 { + return "Score: N/A [measurement not done]" + } + + return fmt.Sprintf(perfMessage, topScale-(network/netScale), topScale-(filesystem/fsScale), seconds, scale, Platform()) +} + +func (it *scorecard) Start() Scorecard { + it.start = time.Now() + return it +} + +func (it *scorecard) Midpoint() Scorecard { + it.network = time.Now() + return it +} + +func (it *scorecard) Done() Scorecard { + it.filesystem = time.Now() + return it +} + +func NewScorecard() Scorecard { + marker := time.Now() + return &scorecard{ + start: marker, + network: marker, + filesystem: marker, + } +} diff --git a/common/strategies.go b/common/strategies.go new file mode 100644 index 00000000..cf7ffebf --- /dev/null +++ b/common/strategies.go @@ -0,0 +1,117 @@ +package common + +import "os" + +const ( + ROBOCORP_HOME_VARIABLE = `ROBOCORP_HOME` + ROBOCORP_NAME = `Robocorp` + SEMA4AI_HOME_VARIABLE = `SEMA4AI_HOME` + SEMA4AI_NAME = `Sema4.ai` +) + +type ( + ProductStrategy interface { + Name() string + IsLegacy() bool + ForceHome(string) + HomeVariable() string + Home() string + HoloLocation() string + DefaultSettingsYamlFile() string + AllowInternalMetrics() bool + } + + legacyStrategy struct { + forcedHome string + } + + sema4Strategy struct { + forcedHome string + } +) + +func LegacyMode() ProductStrategy { + return &legacyStrategy{} +} + +func Sema4Mode() ProductStrategy { + return &sema4Strategy{} +} + +func (it *legacyStrategy) Name() string { + return ROBOCORP_NAME +} + +func (it *legacyStrategy) IsLegacy() bool { + return true +} + +func (it *legacyStrategy) AllowInternalMetrics() bool { + return true +} + +func (it *legacyStrategy) ForceHome(value string) { + it.forcedHome = value +} + +func (it *legacyStrategy) HomeVariable() string { + return ROBOCORP_HOME_VARIABLE +} + +func (it *legacyStrategy) Home() string { + if len(it.forcedHome) > 0 { + return ExpandPath(it.forcedHome) + } + home := os.Getenv(it.HomeVariable()) + if len(home) > 0 { + return ExpandPath(home) + } + return ExpandPath(defaultRobocorpLocation) +} + +func (it *legacyStrategy) HoloLocation() string { + return ExpandPath(defaultHoloLocation) +} + +func (it *legacyStrategy) DefaultSettingsYamlFile() string { + return "assets/robocorp_settings.yaml" +} + +func (it *sema4Strategy) Name() string { + return SEMA4AI_NAME +} + +func (it *sema4Strategy) IsLegacy() bool { + return false +} + +func (it *sema4Strategy) AllowInternalMetrics() bool { + return !IsBundled() +} + +func (it *sema4Strategy) ForceHome(value string) { + it.forcedHome = value +} + +func (it *sema4Strategy) HomeVariable() string { + return SEMA4AI_HOME_VARIABLE +} + +func (it *sema4Strategy) Home() string { + if len(it.forcedHome) > 0 { + return ExpandPath(it.forcedHome) + } + home := os.Getenv(it.HomeVariable()) + if len(home) > 0 { + return ExpandPath(home) + } + return ExpandPath(defaultSema4Location) +} + +func (it *sema4Strategy) HoloLocation() string { + return ExpandPath(defaultSema4HoloLocation) +} + +func (it *sema4Strategy) DefaultSettingsYamlFile() string { + return "assets/sema4ai_settings.yaml" +} diff --git a/common/timeline.go b/common/timeline.go new file mode 100644 index 00000000..a836834b --- /dev/null +++ b/common/timeline.go @@ -0,0 +1,93 @@ +package common + +import ( + "fmt" + "strings" +) + +var ( + TimelineEnabled bool + pipe chan string + indent chan bool + done chan bool +) + +type timevent struct { + level int + when Duration + what string +} + +func timeliner(events chan string, indent, done chan bool) { + history := make([]*timevent, 0, 100) + level := 0 +loop: + for { + select { + case event, ok := <-events: + if !ok { + break loop + } + history = append(history, &timevent{level, Clock.Elapsed(), event}) + case deeper, ok := <-indent: + if !ok { + break loop + } + if deeper { + level += 1 + } else { + level -= 1 + } + if level < 0 { + level = 0 + } + } + } + death := Clock.Elapsed() + if TimelineEnabled && death.Milliseconds() > 0 { + history = append(history, &timevent{0, death, "Now."}) + Log("---- rcc timeline ----") + Log(" # percent seconds event [rcc %s]", Version) + for at, event := range history { + permille := event.when * 1000 / death + percent := float64(permille) / 10.0 + indent := strings.Repeat("| ", event.level) + Log("%3d: %5.1f%% %7s %s%s", at+1, percent, event.when, indent, event.what) + } + Log("---- rcc timeline ----") + } + close(done) +} + +func init() { + pipe = make(chan string) + indent = make(chan bool) + done = make(chan bool) + go timeliner(pipe, indent, done) +} + +func IgnoreAllPanics() { + recover() +} + +func Timeline(form string, details ...interface{}) { + defer IgnoreAllPanics() + pipe <- fmt.Sprintf(form, details...) +} + +func TimelineBegin(form string, details ...interface{}) { + Timeline(form, details...) + indent <- true +} + +func TimelineEnd() { + indent <- false + Timeline("`--") +} + +func EndOfTimeline() { + TimelineEnd() + close(pipe) + close(indent) + <-done +} diff --git a/common/variables.go b/common/variables.go index 5ade2fb8..8caf3f59 100644 --- a/common/variables.go +++ b/common/variables.go @@ -1,22 +1,404 @@ package common -var ( - Silent bool - DebugFlag bool - TraceFlag bool - NoCache bool +import ( + "fmt" + "math/rand" + "os" + "path/filepath" + "runtime" + "strings" + "time" + + "github.com/robocorp/rcc/set" +) + +type ( + Verbosity uint8 +) + +const ( + Undefined Verbosity = 0 + Silently Verbosity = 1 + Normal Verbosity = 2 + Debugging Verbosity = 3 + Tracing Verbosity = 4 ) const ( - DefaultEndpoint = "https://api.eu1.robocloud.eu/" + RCC_REMOTE_ORIGIN = `RCC_REMOTE_ORIGIN` + RCC_REMOTE_AUTHORIZATION = `RCC_REMOTE_AUTHORIZATION` + RCC_NO_TEMP_MANAGEMENT = `RCC_NO_TEMP_MANAGEMENT` + RCC_NO_PYC_MANAGEMENT = `RCC_NO_PYC_MANAGEMENT` + VERBOSE_ENVIRONMENT_BUILDING = `RCC_VERBOSE_ENVIRONMENT_BUILDING` + ROBOCORP_OVERRIDE_SYSTEM_REQUIREMENTS = `ROBOCORP_OVERRIDE_SYSTEM_REQUIREMENTS` + RCC_VERBOSITY = `RCC_VERBOSITY` + SILENTLY = `silent` + TRACING = `trace` + DEBUGGING = `debug` +) + +var ( + NoBuild bool + NoRetryBuild bool + NoTempManagement bool + NoPycManagement bool + ExternallyManaged bool + DeveloperFlag bool + StrictFlag bool + SharedHolotree bool + LogLinenumbers bool + NoCache bool + NoOutputCapture bool + Liveonly bool + UnmanagedSpace bool + FreshlyBuildEnvironment bool + WarrantyVoidedFlag bool + BundledFlag bool + StageFolder string + ControllerType string + HolotreeSpace string + EnvironmentHash string + SemanticTag string + When int64 + Clock *stopwatch + randomIdentifier string + verbosity Verbosity + LogHides []string + Product ProductStrategy ) -func UnifyVerbosityFlags() { - if Silent { - DebugFlag = false - TraceFlag = false +func init() { + Clock = &stopwatch{"Clock", time.Now()} + When = Clock.When() + + randomIdentifier = fmt.Sprintf("%016x", rand.Uint64()^uint64(os.Getpid())) + + lowargs := make([]string, 0, len(os.Args)) + for _, arg := range os.Args { + lowargs = append(lowargs, strings.ToLower(arg)) + } + // peek CLI options to pre-initialize "Warranty Voided" and other indicators + args := set.Set(lowargs) + WarrantyVoidedFlag = set.Member(args, "--warranty-voided") + BundledFlag = set.Member(args, "--bundled") + sema4ai := set.Member(args, "--sema4ai") + robocorp := set.Member(args, "--robocorp") + switch { + case sema4ai && robocorp: + fmt.Fprintln(os.Stderr, "Fatal: rcc cannot be on both --robocorp and --sema4ai product modes at same time! Just use one of those, not both!") + os.Exit(99) + case sema4ai: + Product = Sema4Mode() + case robocorp: + Product = LegacyMode() + default: + Product = LegacyMode() + } + NoTempManagement = set.Member(args, "--no-temp-management") + NoPycManagement = set.Member(args, "--no-pyc-management") + if set.Member(args, "--debug") { + verbosity = Debugging + } + if set.Member(args, "--trace") { + verbosity = Tracing + } + + // Note: HololibCatalogLocation, HololibLibraryLocation and HololibUsageLocation + // are force created from "htfs" direcotry.go init function + // Also: HolotreeLocation creation is left for actual holotree commands + // to prevent accidental access right problem during usage + + SharedHolotree = isFile(HoloInitUserFile()) + + ensureDirectory(JournalLocation()) + ensureDirectory(TemplateLocation()) + ensureDirectory(BinLocation()) + ensureDirectory(UvCache()) + ensureDirectory(PipCache()) + ensureDirectory(WheelCache()) + ensureDirectory(RobotCache()) + ensureDirectory(MambaPackages()) +} + +func RandomIdentifier() string { + return randomIdentifier +} + +func DisableTempManagement() bool { + return NoTempManagement || len(os.Getenv(RCC_NO_TEMP_MANAGEMENT)) > 0 +} + +func DisablePycManagement() bool { + return NoPycManagement || len(os.Getenv(RCC_NO_PYC_MANAGEMENT)) > 0 +} + +func RccRemoteOrigin() string { + return os.Getenv(RCC_REMOTE_ORIGIN) +} + +func RccRemoteAuthorization() (string, bool) { + result := os.Getenv(RCC_REMOTE_AUTHORIZATION) + return result, len(result) > 0 +} + +func ProductLock() string { + return filepath.Join(Product.Home(), "robocorp.lck") +} + +func IsBundled() bool { + return BundledFlag +} + +func WarrantyVoided() bool { + return WarrantyVoidedFlag +} + +func DebugFlag() bool { + return verbosity >= Debugging +} + +func TraceFlag() bool { + return verbosity >= Tracing +} + +func VerboseEnvironmentBuilding() bool { + return DebugFlag() || TraceFlag() || len(os.Getenv(VERBOSE_ENVIRONMENT_BUILDING)) > 0 +} + +func OverrideSystemRequirements() bool { + return len(os.Getenv(ROBOCORP_OVERRIDE_SYSTEM_REQUIREMENTS)) > 0 +} + +func BinRcc() string { + self, err := os.Executable() + if err != nil { + return os.Args[0] + } + return self +} + +func OldEventJournal() string { + return filepath.Join(Product.Home(), "event.log") +} + +func EventJournal() string { + return filepath.Join(JournalLocation(), "event.log") +} + +func JournalLocation() string { + return filepath.Join(Product.Home(), "journals") +} + +func TemplateLocation() string { + return filepath.Join(Product.Home(), "templates") +} + +func ProductTempRoot() string { + return filepath.Join(Product.Home(), "temp") +} + +func ProductTempName() string { + return filepath.Join(ProductTempRoot(), RandomIdentifier()) +} + +func ProductTemp() string { + tempLocation := ProductTempName() + fullpath, err := filepath.Abs(tempLocation) + if err != nil { + fullpath = tempLocation + } + ensureDirectory(fullpath) + if err != nil { + Log("WARNING (%v) -> %v", tempLocation, err) + } + return fullpath +} + +func BinLocation() string { + return filepath.Join(Product.Home(), "bin") +} + +func MicromambaLocation() string { + return filepath.Join(Product.Home(), "micromamba") +} + +func SharedMarkerLocation() string { + return filepath.Join(Product.HoloLocation(), "shared.yes") +} + +func HoloInitLocation() string { + return filepath.Join(Product.HoloLocation(), "lib", "catalog", "init") +} + +func HoloInitUserFile() string { + return filepath.Join(HoloInitLocation(), UserHomeIdentity()) +} + +func HoloInitCommonFile() string { + return filepath.Join(HoloInitLocation(), "commons.tof") +} + +func HolotreeLocation() string { + if SharedHolotree { + return Product.HoloLocation() + } + return filepath.Join(Product.Home(), "holotree") +} + +func HololibLocation() string { + if SharedHolotree { + return filepath.Join(Product.HoloLocation(), "lib") } - if TraceFlag { - DebugFlag = true + return filepath.Join(Product.Home(), "hololib") +} + +func HololibPids() string { + return filepath.Join(HololibLocation(), "pids") +} + +func HololibCatalogLocation() string { + return filepath.Join(HololibLocation(), "catalog") +} + +func HololibLibraryLocation() string { + return filepath.Join(HololibLocation(), "library") +} + +func HololibUsageLocation() string { + return filepath.Join(HololibLocation(), "used") +} + +func HololibCompressMarker() string { + return filepath.Join(HololibCatalogLocation(), "compress.no") +} + +func HolotreeLock() string { + return filepath.Join(HolotreeLocation(), "global.lck") +} + +func BadHololibSitePackagesLocation() string { + return filepath.Join(HololibLocation(), "site-packages") +} + +func BadHololibScriptsLocation() string { + if SharedHolotree { + return filepath.Join(Product.HoloLocation(), "Scripts") + } + return filepath.Join(Product.Home(), "Scripts") +} + +func UsesHolotree() bool { + return len(HolotreeSpace) > 0 +} + +func UvCache() string { + return filepath.Join(Product.Home(), "uvcache") +} + +func PipCache() string { + return filepath.Join(Product.Home(), "pipcache") +} + +func WheelCache() string { + return filepath.Join(Product.Home(), "wheels") +} + +func RobotCache() string { + return filepath.Join(Product.Home(), "robots") +} + +func MambaRootPrefix() string { + return Product.Home() +} + +func MambaPackages() string { + return ExpandPath(filepath.Join(MambaRootPrefix(), "pkgs")) +} + +func PipRcFile() string { + return ExpandPath(filepath.Join(Product.Home(), "piprc")) +} + +func MicroMambaRcFile() string { + return ExpandPath(filepath.Join(Product.Home(), "micromambarc")) +} + +func SettingsFile() string { + return ExpandPath(filepath.Join(Product.Home(), "settings.yaml")) +} + +func CaBundleFile() string { + return ExpandPath(filepath.Join(Product.Home(), "ca-bundle.pem")) +} + +func DefineVerbosity(silent, debug, trace bool) { + override := os.Getenv(RCC_VERBOSITY) + switch { + case silent || override == SILENTLY: + verbosity = Silently + case trace || override == TRACING: + verbosity = Tracing + case debug || override == DEBUGGING: + verbosity = Debugging + default: + verbosity = Normal + } +} + +func Silent() bool { + return verbosity == Silently +} + +func UnifyStageHandling() { + if len(StageFolder) > 0 { + Liveonly = true + } +} + +func ForceDebug() { + DefineVerbosity(false, true, false) +} + +func Platform() string { + return strings.ToLower(fmt.Sprintf("%s_%s", runtime.GOOS, runtime.GOARCH)) +} + +func UserAgent() string { + return fmt.Sprintf("rcc/%s (%s %s) %s", Version, runtime.GOOS, runtime.GOARCH, ControllerIdentity()) +} + +func ControllerIdentity() string { + return strings.ToLower(fmt.Sprintf("rcc.%s", ControllerType)) +} + +func isFile(pathname string) bool { + stat, err := os.Stat(pathname) + return err == nil && stat.Mode().IsRegular() +} + +func isDir(pathname string) bool { + stat, err := os.Stat(pathname) + return err == nil && stat.IsDir() +} + +func ensureDirectory(name string) { + if !WarrantyVoided() && !isDir(name) { + Error("mkdir", os.MkdirAll(name, 0o750)) + } +} + +func SymbolicUserIdentity() string { + location, err := os.UserHomeDir() + if err != nil { + return "badcafe" + } + digest := fmt.Sprintf("%02x", Siphash(9007799254740993, 2147487647, []byte(location))) + return digest[:7] +} + +func UserHomeIdentity() string { + if UnmanagedSpace { + return "UNMNGED" } + return SymbolicUserIdentity() } diff --git a/common/version.go b/common/version.go index bb051ef3..41303060 100644 --- a/common/version.go +++ b/common/version.go @@ -1,5 +1,5 @@ package common const ( - Version = `v6.0.6` + Version = `v18.1.7` ) diff --git a/conda/activate.go b/conda/activate.go new file mode 100644 index 00000000..fa7249bf --- /dev/null +++ b/conda/activate.go @@ -0,0 +1,166 @@ +package conda + +import ( + "bytes" + "encoding/json" + "fmt" + "html/template" + "io" + "os" + "path/filepath" + "strings" + + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/pathlib" +) + +const ( + preformatMarker = "```" + activateFile = "rcc_activate.json" +) + +func capturePreformatted(incoming string) ([]string, string) { + lines := strings.SplitAfter(incoming, "\n") + capture := false + result := make([]string, 0, 5) + pending := make([]string, 0, len(lines)) + other := make([]string, 0, len(lines)) + for _, line := range lines { + flat := strings.TrimSpace(line) + if strings.HasPrefix(flat, preformatMarker) { + if len(pending) > 0 { + result = append(result, strings.Join(pending, "")) + pending = make([]string, 0, len(lines)) + } + capture = !capture + continue + } + if !capture { + other = append(other, line) + continue + } + pending = append(pending, line) + } + if len(pending) > 0 { + result = append(result, strings.Join(pending, "")) + } + return result, strings.Join(other, "") +} + +func createScript(targetFolder string) (string, error) { + script := template.New("script") + script, err := script.Parse(activateScript) + if err != nil { + return "", err + } + details := make(map[string]string) + details["Rcc"] = common.BinRcc() + details["Robocorphome"] = common.Product.Home() + details["MambaRootPrefix"] = common.MambaRootPrefix() + details["Micromamba"] = BinMicromamba() + details["Live"] = targetFolder + buffer := bytes.NewBuffer(nil) + script.Execute(buffer, details) + + scriptfile := filepath.Join(targetFolder, fmt.Sprintf("rcc_activate%s", commandSuffix)) + err = pathlib.WriteFile(scriptfile, buffer.Bytes(), 0o755) + if err != nil { + return "", err + } + return scriptfile, nil +} + +func parseJson(content string) (map[string]string, error) { + result := make(map[string]string) + err := json.Unmarshal([]byte(content), &result) + return result, err +} + +func diffStringMaps(before, after map[string]string) map[string]string { + result := make(map[string]string) + for key, _ := range before { + _, ok := after[key] + if !ok { + result[key] = "" + } + } + for key, past := range before { + future, ok := after[key] + if ok && past != future { + result[key] = future + } + } + for key, value := range after { + _, ok := before[key] + if !ok { + result[key] = value + } + } + return result +} + +func Activate(sink io.Writer, targetFolder string) error { + envCommand := []string{common.BinRcc(), "internal", "env", "--label", "before"} + out, _, err := LiveCapture(targetFolder, envCommand...) + if err != nil { + return err + } + parts, _ := capturePreformatted(out) + if len(parts) == 0 { + return fmt.Errorf("Could not detect environment details from 'before' output.") + } + before, err := parseJson(parts[0]) + if err != nil { + return err + } + + script, err := createScript(targetFolder) + if err != nil { + return err + } + + out, _, err = LiveCapture(targetFolder, script) + if err != nil { + fmt.Fprintf(sink, "%v\n%s\n", err, out) + return err + } + parts, other := capturePreformatted(out) + fmt.Fprintf(sink, "%s\n", other) + if len(parts) == 0 { + return fmt.Errorf("Could not detect environment details from 'after' output.") + } + after, err := parseJson(parts[0]) + if err != nil { + return err + } + difference := diffStringMaps(before, after) + body, err := json.MarshalIndent(difference, "", " ") + if err != nil { + return err + } + targetJson := filepath.Join(targetFolder, activateFile) + err = pathlib.WriteFile(targetJson, body, 0o644) + if err != nil { + return err + } + return nil +} + +func LoadActivationEnvironment(targetFolder string) []string { + result := []string{} + targetJson := filepath.Join(targetFolder, activateFile) + content, err := os.ReadFile(targetJson) + if err != nil { + return result + } + var entries map[string]string + err = json.Unmarshal(content, &entries) + if err != nil { + return result + } + for name, value := range entries { + result = append(result, fmt.Sprintf("%s=%s", name, value)) + } + common.Trace("Environment activation added %d variables.", len(result)) + return result +} diff --git a/conda/cleanup.go b/conda/cleanup.go index 0f2f3aba..ce030ff3 100644 --- a/conda/cleanup.go +++ b/conda/cleanup.go @@ -1,34 +1,233 @@ package conda import ( + "os" + "path/filepath" "time" "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/fail" + "github.com/robocorp/rcc/pathlib" + "github.com/robocorp/rcc/pretty" ) -func Cleanup(daylimit int, dryrun, all bool) error { - deadline := time.Now().Add(-24 * time.Duration(daylimit) * time.Hour) - for _, template := range TemplateList() { - whenLive, err := LastUsed(LiveFrom(template)) - if err != nil { - return err - } - if !all && whenLive.After(deadline) { - continue - } - whenBase, err := LastUsed(TemplateFrom(template)) - if err != nil { - return err - } - if !all && whenBase.After(deadline) { +func safeRemove(hint, pathling string) error { + var err error + if !pathlib.Exists(pathling) { + common.Debug("[%s] Missing %v, no need to remove.", hint, pathling) + return nil + } + if pathlib.IsDir(pathling) { + err = renameRemove(pathling) + } else { + err = os.Remove(pathling) + } + if err != nil { + pretty.Warning("[%s] %s -> %v", hint, pathling, err) + pretty.Warning("Make sure that you have rights to %q, and that nothing has locks in there.", pathling) + } else { + common.Debug("[%s] Removed %v.", hint, pathling) + } + return err +} + +func doCleanup(fullpath string, dryrun bool) error { + if !pathlib.Exists(fullpath) { + return nil + } + if dryrun { + common.Log("Would be removing: %s", fullpath) + return nil + } + return safeRemove("path", fullpath) +} + +func bugsCleanup(dryrun bool) { + if dryrun { + common.Log("- %v", common.BadHololibSitePackagesLocation()) + common.Log("- %v", common.BadHololibScriptsLocation()) + return + } + safeRemove("bugs", common.BadHololibSitePackagesLocation()) + safeRemove("bugs", common.BadHololibScriptsLocation()) +} + +func alwaysCleanup(dryrun bool) { + base := filepath.Join(common.Product.Home(), "base") + live := filepath.Join(common.Product.Home(), "live") + miniconda3 := filepath.Join(common.Product.Home(), "miniconda3") + if dryrun { + common.Log("Would be removing:") + common.Log("- %v", base) + common.Log("- %v", live) + common.Log("- %v", miniconda3) + return + } + safeRemove("legacy", base) + safeRemove("legacy", live) + safeRemove("legacy", miniconda3) +} + +func downloadCleanup(dryrun bool) (err error) { + defer fail.Around(&err) + if dryrun { + common.Log("- %v", common.TemplateLocation()) + common.Log("- %v", common.PipCache()) + common.Log("- %v", common.UvCache()) + common.Log("- %v", common.MambaPackages()) + } else { + fail.Fast(safeRemove("templates", common.TemplateLocation())) + fail.Fast(safeRemove("cache", common.PipCache())) + fail.Fast(safeRemove("cache", common.UvCache())) + fail.Fast(safeRemove("cache", common.MambaPackages())) + } + return nil +} + +func quickCleanup(dryrun bool) error { + downloadCleanup(dryrun) + if dryrun { + common.Log("- %v", common.HolotreeLocation()) + common.Log("- %v", common.ProductTempRoot()) + return nil + } + err := safeRemove("cache", common.HolotreeLocation()) + if err != nil { + return err + } + return safeRemove("temp", common.ProductTempRoot()) +} + +func cleanupAllCaches(dryrun bool) error { + downloadCleanup(dryrun) + if dryrun { + common.Log("- %v", common.HololibLocation()) + return nil + } + fail.Fast(safeRemove("cache", common.HololibLocation())) + return nil +} + +func spotlessCleanup(dryrun, noCompress bool) (err error) { + defer fail.Around(&err) + + fail.Fast(quickCleanup(dryrun)) + rcccache := filepath.Join(common.Product.Home(), "rcccache.yaml") + if dryrun { + common.Log("- %v", common.BinLocation()) + common.Log("- %v", common.MicromambaLocation()) + common.Log("- %v", common.RobotCache()) + common.Log("- %v", rcccache) + common.Log("- %v", common.OldEventJournal()) + common.Log("- %v", common.JournalLocation()) + common.Log("- %v", common.HololibCatalogLocation()) + common.Log("- %v", common.HololibLocation()) + return nil + } + fail.Fast(safeRemove("executables", common.BinLocation())) + fail.Fast(safeRemove("micromamba", common.MicromambaLocation())) + fail.Fast(safeRemove("cache", common.RobotCache())) + fail.Fast(safeRemove("cache", rcccache)) + fail.Fast(safeRemove("old", common.OldEventJournal())) + fail.Fast(safeRemove("journals", common.JournalLocation())) + fail.Fast(safeRemove("catalogs", common.HololibCatalogLocation())) + fail.Fast(safeRemove("cache", common.HololibLocation())) + if noCompress { + return pathlib.WriteFile(common.HololibCompressMarker(), []byte("present"), 0o666) + } + return nil +} + +func cleanupTemp(deadline time.Time, dryrun bool) error { + basedir := common.ProductTempRoot() + handle, err := os.Open(basedir) + if err != nil { + return err + } + entries, err := handle.Readdir(-1) + handle.Close() + if err != nil { + return err + } + for _, entry := range entries { + if entry.ModTime().After(deadline) { continue } + fullpath := filepath.Join(basedir, entry.Name()) if dryrun { - common.Log("Would be removing %v.", template) + common.Log("Would remove temp %v.", fullpath) continue } - RemoveEnvironment(template) - common.Debug("Removed environment %v.", template) + if entry.IsDir() { + err = os.RemoveAll(fullpath) + if err != nil { + common.Log("Warning[%q]: %v", fullpath, err) + } + } else { + os.Remove(fullpath) + } + common.Debug("Removed %v.", fullpath) } return nil } + +func BugsCleanup() { + bugsCleanup(false) +} + +func Cleanup(daylimit int, dryrun, quick, all, micromamba, downloads, noCompress, caches bool) (err error) { + defer fail.Around(&err) + + lockfile := common.ProductLock() + completed := pathlib.LockWaitMessage(lockfile, "Serialized environment cleanup [robocorp lock]") + locker, err := pathlib.Locker(lockfile, 30000, false) + completed() + if err != nil { + common.Log("Could not get lock on live environment. Quitting!") + return err + } + defer locker.Release() + + alwaysCleanup(dryrun) + bugsCleanup(dryrun) + + if downloads { + return downloadCleanup(dryrun) + } + + if quick { + return quickCleanup(dryrun) + } + + if caches { + fail.Fast(cleanupAllCaches(dryrun)) + } + + if all { + return spotlessCleanup(dryrun, noCompress) + } + + deadline := time.Now().Add(-24 * time.Duration(daylimit) * time.Hour) + cleanupTemp(deadline, dryrun) + + if micromamba && err == nil { + err = doCleanup(common.MambaPackages(), dryrun) + } + if micromamba && err == nil { + err = doCleanup(BinMicromamba(), dryrun) + } + if micromamba && err == nil { + err = doCleanup(common.MicromambaLocation(), dryrun) + } + return err +} + +func RemoveCurrentTemp() { + target := common.ProductTempName() + common.Debug("removing current temp %v", target) + common.Timeline("removing current temp: %v", target) + err := safeRemove("temp", target) + if err != nil { + common.Timeline("removing current temp failed, reason: %v", err) + } +} diff --git a/conda/condayaml.go b/conda/condayaml.go index b40a80a3..e4af3154 100644 --- a/conda/condayaml.go +++ b/conda/condayaml.go @@ -1,20 +1,30 @@ package conda import ( - "errors" "fmt" - "io/ioutil" + "os" "regexp" "strings" + "github.com/robocorp/rcc/cloud" "github.com/robocorp/rcc/common" "github.com/robocorp/rcc/pathlib" "gopkg.in/yaml.v2" ) +const ( + useFeatureTruststore = `--use-feature=truststore` + alternative = `|` + reject_chars = `(?:[][(){}%/:,;@*<=>!]+)` + and_or = `\b(?:or|and)\b` + dash_start = `^-+` + uncacheableForm = reject_chars + alternative + and_or + alternative + dash_start +) + var ( - dependencyPattern = regexp.MustCompile("^([^<=~!> ]+)\\s*(?:([<=~!>]*)\\s*(\\S+.*?))?$") + dependencyPattern = regexp.MustCompile("^([^<=~!> ]+)\\s*(?:([<=~!>]*)\\s*(\\S+.*?))?$") + uncacheablePattern = regexp.MustCompile(uncacheableForm) ) type internalEnvironment struct { @@ -22,14 +32,16 @@ type internalEnvironment struct { Channels []string `yaml:"channels"` Dependencies []interface{} `yaml:"dependencies"` Prefix string `yaml:"prefix,omitempty"` + PostInstall []string `yaml:"rccPostInstall,omitempty"` } type Environment struct { - Name string - Prefix string - Channels []string - Conda []*Dependency - Pip []*Dependency + Name string + Prefix string + Channels []string + Conda []*Dependency + Pip []*Dependency + PostInstall []string } type Dependency struct { @@ -53,12 +65,47 @@ func AsDependency(value string) *Dependency { } } +func IsSpecialCacheable(name, version string) bool { + flat := fmt.Sprintf("%s=%s", strings.TrimSpace(name), strings.TrimSpace(version)) + return strings.EqualFold(flat, useFeatureTruststore) +} + +func IsCacheable(text string) bool { + flat := strings.TrimSpace(text) + return !uncacheablePattern.MatchString(flat) +} + +func (it *Dependency) Representation() string { + parts := strings.SplitN(strings.ToLower(it.Name), "[", 2) + return parts[0] +} + +func (it *Dependency) IsCacheable() bool { + if IsSpecialCacheable(it.Name, it.Versions) { + return true + } + if !it.IsExact() { + return false + } + if !IsCacheable(it.Name) { + return false + } + if !IsCacheable(it.Versions) { + return false + } + return true +} + +func (it *Dependency) Match(name string) bool { + return strings.EqualFold(name, it.Name) +} + func (it *Dependency) IsExact() bool { return len(it.Qualifier)+len(it.Versions) > 0 } func (it *Dependency) SameAs(right *Dependency) bool { - return it.Name == right.Name + return !strings.HasPrefix(it.Name, "-") && it.Name == right.Name } func (it *Dependency) ExactlySame(right *Dependency) bool { @@ -67,7 +114,7 @@ func (it *Dependency) ExactlySame(right *Dependency) bool { func (it *Dependency) ChooseSpecific(right *Dependency) (*Dependency, error) { if !it.SameAs(right) { - return nil, errors.New(fmt.Sprintf("Not same component: %v vs. %v", it.Name, right.Name)) + return nil, fmt.Errorf("Not same component: %v vs. %v", it.Name, right.Name) } if it.IsExact() && !right.IsExact() { return it, nil @@ -78,7 +125,7 @@ func (it *Dependency) ChooseSpecific(right *Dependency) (*Dependency, error) { if it.ExactlySame(right) { return it, nil } - return nil, errors.New(fmt.Sprintf("Wont choose between dependencies: %v vs. %v", it.Original, right.Original)) + return nil, fmt.Errorf("Wont choose between dependencies: %v vs. %v", it.Original, right.Original) } func (it *Dependency) Index(others []*Dependency) int { @@ -92,9 +139,12 @@ func (it *Dependency) Index(others []*Dependency) int { func (it *internalEnvironment) AsEnvironment() *Environment { result := &Environment{ - Name: it.Name, - Prefix: it.Prefix, + Name: it.Name, + Prefix: it.Prefix, + PostInstall: []string{}, } + seenScripts := make(map[string]bool) + result.PostInstall = addItem(seenScripts, it.PostInstall, result.PostInstall) channel, ok := LocalChannel() if ok { pushChannels(result, []string{channel}) @@ -113,18 +163,127 @@ func remove(index int, target []*Dependency) []*Dependency { func SummonEnvironment(filename string) *Environment { if pathlib.IsFile(filename) { - result, err := ReadCondaYaml(filename) + result, err := ReadPackageCondaYaml(filename) if err == nil { return result } } return &Environment{ - Channels: []string{"defaults", "conda-forge"}, + Channels: []string{"conda-forge"}, Conda: []*Dependency{}, Pip: []*Dependency{}, } } +func (it *Environment) HasCondaDependency(name string) bool { + for _, dependency := range it.Conda { + if dependency.Match(name) { + return true + } + } + return false +} + +func (it *Environment) IsCacheable() bool { + for _, dependency := range it.Conda { + if !dependency.IsCacheable() { + return false + } + } + for _, dependency := range it.Pip { + if !dependency.IsCacheable() { + return false + } + } + return true +} + +func (it *Environment) FreezeDependencies(fixed dependencies) *Environment { + result := &Environment{ + Name: it.Name, + Prefix: it.Prefix, + Channels: it.Channels, + Conda: []*Dependency{}, + Pip: []*Dependency{}, + PostInstall: it.PostInstall, + } + used := make(map[string]bool) + for _, dependency := range fixed { + if dependency.Origin == "pypi" { + continue + } + if used[dependency.Name] { + continue + } + used[dependency.Name] = true + result.Conda = append(result.Conda, &Dependency{ + Original: fmt.Sprintf("%s=%s", dependency.Name, dependency.Version), + Name: dependency.Name, + Qualifier: "=", + Versions: dependency.Version, + }) + } + for _, dependency := range fixed { + if dependency.Origin != "pypi" { + continue + } + if used[dependency.Name] { + continue + } + used[dependency.Name] = true + result.Pip = append(result.Pip, &Dependency{ + Original: fmt.Sprintf("%s==%s", dependency.Name, dependency.Version), + Name: dependency.Name, + Qualifier: "==", + Versions: dependency.Version, + }) + } + return result +} + +func (it *Environment) FromDependencies(fixed dependencies) (*Environment, bool) { + result := &Environment{ + Name: it.Name, + Prefix: it.Prefix, + Channels: it.Channels, + Conda: []*Dependency{}, + Pip: []*Dependency{}, + PostInstall: it.PostInstall, + } + same := true + for _, dependency := range it.Conda { + found, ok := fixed.Lookup(dependency.Name, false) + if !ok { + result.Conda = append(result.Conda, dependency) + same = false + common.Debug("Could not fix version for dependency %q from conda.", dependency.Name) + continue + } + result.Conda = append(result.Conda, &Dependency{ + Original: fmt.Sprintf("%s=%s", dependency.Name, found.Version), + Name: dependency.Name, + Qualifier: "=", + Versions: found.Version, + }) + } + for _, dependency := range it.Pip { + found, ok := fixed.Lookup(dependency.Name, true) + if !ok { + result.Conda = append(result.Pip, dependency) + same = false + common.Debug("Could not fix version for dependency %q from pypi.", dependency.Name) + continue + } + result.Pip = append(result.Pip, &Dependency{ + Original: fmt.Sprintf("%s==%s", dependency.Name, found.Version), + Name: dependency.Name, + Qualifier: "==", + Versions: found.Version, + }) + } + return result, same +} + func (it *Environment) pipPromote() error { removed := make([]int, 0, len(it.Pip)) @@ -180,12 +339,12 @@ func (it *internalEnvironment) condaDependencies() []*Dependency { return result } -func addChannels(seen map[string]bool, source, target []string) []string { - for _, channel := range source { - found := seen[channel] +func addItem(seen map[string]bool, source, target []string) []string { + for _, item := range source { + found := seen[item] if !found { - seen[channel] = true - target = append(target, channel) + seen[item] = true + target = append(target, item) } } return target @@ -219,10 +378,18 @@ func pushPip(target *Environment, dependencies []*Dependency) error { func (it *Environment) Merge(right *Environment) (*Environment, error) { result := new(Environment) - result.Name = it.Name + "+" + right.Name + if len(it.Name) > 0 || len(right.Name) > 0 { + result.Name = it.Name + "+" + right.Name + } + seenChannels := make(map[string]bool) - result.Channels = addChannels(seenChannels, it.Channels, result.Channels) - result.Channels = addChannels(seenChannels, right.Channels, result.Channels) + result.Channels = addItem(seenChannels, it.Channels, result.Channels) + result.Channels = addItem(seenChannels, right.Channels, result.Channels) + + seenScripts := make(map[string]bool) + result.PostInstall = addItem(seenScripts, it.PostInstall, result.PostInstall) + result.PostInstall = addItem(seenScripts, right.PostInstall, result.PostInstall) + err := pushConda(result, it.Conda) if err != nil { return nil, err @@ -312,11 +479,23 @@ func (it *Environment) PipMap() map[interface{}]interface{} { func (it *Environment) AsPureConda() *Environment { return &Environment{ - Name: it.Name, - Prefix: it.Prefix, - Channels: it.Channels, - Conda: it.Conda, - Pip: []*Dependency{}, + Name: it.Name, + Prefix: it.Prefix, + Channels: it.Channels, + Conda: it.Conda, + Pip: []*Dependency{}, + PostInstall: []string{}, + } +} + +func (it *Environment) WithoutPostInstall() *Environment { + return &Environment{ + Name: it.Name, + Prefix: it.Prefix, + Channels: it.Channels, + Conda: it.Conda, + Pip: it.Pip, + PostInstall: []string{}, } } @@ -325,12 +504,14 @@ func (it *Environment) SaveAs(filename string) error { if err != nil { return err } - return ioutil.WriteFile(filename, []byte(content), 0o640) + common.Trace("FINAL conda environment file as %v:\n---\n%v---", filename, content) + return pathlib.WriteFile(filename, []byte(content), 0o640) } func (it *Environment) SaveAsRequirements(filename string) error { content := it.AsRequirementsText() - return ioutil.WriteFile(filename, []byte(content), 0o640) + common.Trace("FINAL pip requirements as %v:\n---\n%v\n---", filename, content) + return pathlib.WriteFile(filename, []byte(content), 0o640) } func (it *Environment) AsYaml() (string, error) { @@ -339,6 +520,8 @@ func (it *Environment) AsYaml() (string, error) { result.Prefix = it.Prefix result.Channels = it.Channels result.Dependencies = it.CondaList() + seenScripts := make(map[string]bool) + result.PostInstall = addItem(seenScripts, it.PostInstall, result.PostInstall) if len(it.Pip) > 0 { result.Dependencies = append(result.Dependencies, it.PipMap()) } @@ -357,6 +540,115 @@ func (it *Environment) AsRequirementsText() string { return strings.Join(lines, Newline) } +func (it *Environment) AsLayers() [3]string { + conda, _ := it.AsPureConda().AsYaml() + pip, _ := it.WithoutPostInstall().AsYaml() + full, _ := it.AsYaml() + return [3]string{ + strings.TrimSpace(conda), + strings.TrimSpace(pip), + strings.TrimSpace(full), + } +} + +func (it *Environment) FingerprintLayers() [3]string { + layers := it.AsLayers() + return [3]string{ + common.BlueprintHash([]byte(layers[0])), + common.BlueprintHash([]byte(layers[1])), + common.BlueprintHash([]byte(layers[2])), + } +} + +func (it *Environment) Diagnostics(target *common.DiagnosticStatus, production bool) { + target.Details["cacheable-environment-configuration"] = fmt.Sprintf("%v", it.IsCacheable()) + + diagnose := target.Diagnose("Conda") + notice := diagnose.Warning + if production { + notice = diagnose.Fail + } + packages := make(map[string]bool) + countChannels := len(it.Channels) + defaultsPostion := -1 + floating := false + ok := true + for index, channel := range it.Channels { + if channel == "defaults" { + defaultsPostion = index + diagnose.Warning(0, "", "Try to avoid defaults channel, and prefer using conda-forge instead.") + ok = false + } + } + if defaultsPostion == 0 && countChannels > 1 { + diagnose.Warning(0, "", "Try to avoid putting defaults channel as first channel.") + ok = false + } + if countChannels > 1 { + diagnose.Warning(0, "", "Try to avoid multiple channel. They may cause problems with code compatibility.") + ok = false + } + if ok { + diagnose.Ok(0, "Channels in conda.yaml are ok.") + } + ok = true + for _, dependency := range it.Conda { + presentation := dependency.Representation() + if packages[presentation] { + notice(0, "", "Dependency %q seems to be duplicate of previous dependency.", dependency.Original) + } + packages[presentation] = true + if !dependency.IsCacheable() { + diagnose.Warning(common.CategoryEnvironmentCache, "", "Conda dependency %q is not publicly cacheable.", dependency.Original) + ok = false + } + if strings.Contains(dependency.Versions, "*") || len(dependency.Qualifier) == 0 || len(dependency.Versions) == 0 { + notice(0, "", "Floating conda dependency %q should be bound to exact version before taking robot into production.", dependency.Original) + ok = false + floating = true + } + if len(dependency.Qualifier) > 0 && !(dependency.Qualifier == "==" || dependency.Qualifier == "=") { + diagnose.Fail(0, "", "Conda dependency %q must use '==' or '=' for version declaration.", dependency.Original) + ok = false + floating = true + } + } + if ok { + diagnose.Ok(0, "Conda dependencies in conda.yaml are ok.") + } + ok = true + for _, dependency := range it.Pip { + if IsSpecialCacheable(dependency.Name, dependency.Versions) { + continue + } + presentation := dependency.Representation() + if packages[presentation] { + notice(0, "", "Dependency %q seems to be duplicate of previous dependency.", dependency.Original) + } + packages[presentation] = true + if !dependency.IsCacheable() { + diagnose.Warning(common.CategoryEnvironmentCache, "", "Pip dependency %q is not publicly cacheable.", dependency.Original) + ok = false + } + if strings.Contains(dependency.Versions, "*") || len(dependency.Qualifier) == 0 || len(dependency.Versions) == 0 { + notice(0, "", "Floating pip dependency %q should be bound to exact version before taking robot into production.", dependency.Original) + ok = false + floating = true + } + if len(dependency.Qualifier) > 0 && dependency.Qualifier != "==" { + diagnose.Fail(0, "", "Pip dependency %q must use '==' for version declaration.", dependency.Original) + ok = false + floating = true + } + } + if ok { + diagnose.Ok(0, "Pip dependencies in conda.yaml are ok.") + } + if floating { + diagnose.Warning(0, "", "Floating dependencies in %s Cloud containers will be slow, because floating environments cannot be cached.", common.Product.Name()) + } +} + func CondaYamlFrom(content []byte) (*Environment, error) { result := new(internalEnvironment) err := yaml.Unmarshal(content, result) @@ -366,10 +658,17 @@ func CondaYamlFrom(content []byte) (*Environment, error) { return result.AsEnvironment(), nil } -func ReadCondaYaml(filename string) (*Environment, error) { - content, err := ioutil.ReadFile(filename) +func readCondaYaml(filename string) (*Environment, error) { + var content []byte + var err error + + if pathlib.IsFile(filename) { + content, err = os.ReadFile(filename) + } else { + content, err = cloud.ReadFile(filename) + } if err != nil { - return nil, err + return nil, fmt.Errorf("%q: %w", filename, err) } return CondaYamlFrom(content) } diff --git a/conda/condayaml_test.go b/conda/condayaml_test.go index a4ca109a..e7ebebdb 100644 --- a/conda/condayaml_test.go +++ b/conda/condayaml_test.go @@ -3,6 +3,7 @@ package conda_test import ( "testing" + "github.com/robocorp/rcc/common" "github.com/robocorp/rcc/conda" "github.com/robocorp/rcc/hamlet" ) @@ -15,7 +16,7 @@ func TestCanParseDependencies(t *testing.T) { must_be.Equal("python", conda.AsDependency("python").Name) must_be.Equal("", conda.AsDependency("python").Qualifier) must_be.Equal("", conda.AsDependency("python").Versions) - wont_be.Nil(conda.AsDependency("python=3.7.5")) + wont_be.Nil(conda.AsDependency("python=3.9.13")) must_be.Equal("python=3.7.4", conda.AsDependency("python=3.7.4").Original) must_be.Equal("python", conda.AsDependency("python=3.7.4").Name) must_be.Equal("=", conda.AsDependency("python=3.7.4").Qualifier) @@ -28,7 +29,7 @@ func TestCanCompareDependencies(t *testing.T) { first := conda.AsDependency("python") second := conda.AsDependency("python=3.7.7") - third := conda.AsDependency("python=3.7.5") + third := conda.AsDependency("python=3.9.13") fourth := conda.AsDependency("robotframework=3.2") wont_be.True(first.IsExact()) @@ -55,7 +56,7 @@ func TestCanCompareDependencies(t *testing.T) { chosen, err = second.ChooseSpecific(third) wont_be.Nil(err) - must_be.Equal("Wont choose between dependencies: python=3.7.7 vs. python=3.7.5", err.Error()) + must_be.Equal("Wont choose between dependencies: python=3.7.7 vs. python=3.9.13", err.Error()) must_be.Nil(chosen) } @@ -70,12 +71,13 @@ func TestCanCreateCondaYamlFromEmptyByteSlice(t *testing.T) { must_be.Equal(0, len(sut.Channels)) must_be.Equal(0, len(sut.Conda)) must_be.Equal(0, len(sut.Pip)) + must_be.Equal(0, len(sut.PostInstall)) } -func TestCanReadCondaYaml(t *testing.T) { +func TestCanReadPackageCondaYaml(t *testing.T) { must_be, wont_be := hamlet.Specifications(t) - sut, err := conda.ReadCondaYaml("testdata/conda.yaml") + sut, err := conda.ReadPackageCondaYaml("testdata/conda.yaml") must_be.Nil(err) wont_be.Nil(sut) must_be.Equal("", sut.Name) @@ -88,16 +90,16 @@ func TestCanReadCondaYaml(t *testing.T) { func TestCanMergeTwoEnvironments(t *testing.T) { must_be, wont_be := hamlet.Specifications(t) - left, err := conda.ReadCondaYaml("testdata/third.yaml") + left, err := conda.ReadPackageCondaYaml("testdata/third.yaml") must_be.Nil(err) wont_be.Nil(left) - right, err := conda.ReadCondaYaml("testdata/other.yaml") + right, err := conda.ReadPackageCondaYaml("testdata/other.yaml") must_be.Nil(err) wont_be.Nil(right) sut, err := left.Merge(right) must_be.Nil(err) wont_be.Nil(sut) - must_be.Equal("+", sut.Name) + must_be.Equal("", sut.Name) must_be.Equal(2, len(sut.Channels)) must_be.Equal(4, len(sut.Conda)) must_be.Equal(1, len(sut.Pip)) @@ -116,3 +118,62 @@ func TestCanCreateEmptyEnvironment(t *testing.T) { sut := conda.SummonEnvironment("tmp/missing.yaml") wont_be.Nil(sut) } + +func TestCanGetLayersFromCondaYaml(t *testing.T) { + must_be, wont_be := hamlet.Specifications(t) + + sut, err := conda.ReadPackageCondaYaml("testdata/layers.yaml") + must_be.Nil(err) + wont_be.Nil(sut) + + layers := sut.AsLayers() + wont_be.Nil(layers) + wont_be.Equal(len(layers[0]), 0) + must_be.True(len(layers[0]) < len(layers[1])) + must_be.True(len(layers[1]) < len(layers[2])) + wont_be.Equal(layers[0], layers[1]) + wont_be.Equal(layers[0], layers[2]) + wont_be.Equal(layers[1], layers[2]) + + must_be.Equal("0d8cc85130420984", common.BlueprintHash([]byte(layers[0]))) + must_be.Equal("5be3e197c8c2c67d", common.BlueprintHash([]byte(layers[1]))) + must_be.Equal("d310697aca0840a1", common.BlueprintHash([]byte(layers[2]))) + + fingerprints := sut.FingerprintLayers() + must_be.Equal("0d8cc85130420984", fingerprints[0]) + must_be.Equal("5be3e197c8c2c67d", fingerprints[1]) + must_be.Equal("d310697aca0840a1", fingerprints[2]) +} + +func TestCacheability(t *testing.T) { + must_be, wont_be := hamlet.Specifications(t) + + // some are from https://peps.python.org/pep-0508/ + + must_be.True(conda.IsCacheable("A.B-C_D")) + must_be.True(conda.IsCacheable("simple")) + must_be.True(conda.IsCacheable("simple space separated")) // by itself, ok + must_be.True(conda.IsCacheable("simple-parts")) + must_be.True(conda.IsCacheable("simple_parts")) + must_be.True(conda.IsCacheable("1.2.3")) + must_be.True(conda.IsCacheable("2023c")) + must_be.True(conda.IsCacheable("2023.3")) + must_be.True(conda.IsCacheable("0.1.0.post0")) + must_be.True(conda.IsSpecialCacheable("--use-feature", "truststore")) + + wont_be.True(conda.IsCacheable("a,b")) + wont_be.True(conda.IsCacheable("simple or not")) + wont_be.True(conda.IsCacheable("simple and other")) + wont_be.True(conda.IsCacheable("-simple")) + wont_be.True(conda.IsCacheable(" -simple")) + wont_be.True(conda.IsCacheable("-c constraints.txt")) + wont_be.True(conda.IsCacheable("-r requirements.txt")) + wont_be.True(conda.IsCacheable("simple*")) + wont_be.True(conda.IsCacheable("3.5.*")) + wont_be.True(conda.IsCacheable("name@http://foo.com")) + wont_be.True(conda.IsCacheable("requests[security]")) + wont_be.True(conda.IsCacheable("./downloads/numpy-1.9.2-cp34-none-win32.whl")) + wont_be.True(conda.IsCacheable("urllib3 @ https://github.com/urllib3/urllib3/archive/refs/tags/1.26.8.zip")) + wont_be.True(conda.IsCacheable("urllib3@https://github.com/urllib3/urllib3/archive/refs/tags/1.26.8.zip")) + wont_be.True(conda.IsCacheable("https://github.com/urllib3/urllib3/archive/refs/tags/1.26.8.zip")) +} diff --git a/conda/config.go b/conda/config.go index 75696c2b..98abf002 100644 --- a/conda/config.go +++ b/conda/config.go @@ -1,13 +1,10 @@ package conda import ( - "errors" - "io/ioutil" + "os" "regexp" "sort" "strings" - - "github.com/glaslos/tlsh" ) var ( @@ -23,20 +20,13 @@ func SplitLines(value string) []string { } func ReadConfig(filename string) (string, error) { - content, err := ioutil.ReadFile(filename) + content, err := os.ReadFile(filename) if err != nil { return "", err } return string(content), nil } -func LocalitySensitiveHash(parts []string) (string, error) { - content := "==================================================\n" - content += strings.Join(parts, "\n") - result, err := tlsh.HashBytes([]byte(content)) - return result.String(), err -} - func AsUnifiedLines(value string) []string { parts := SplitLines(value) limit := len(parts) @@ -55,27 +45,3 @@ func AsUnifiedLines(value string) []string { sort.Strings(result) return result } - -func HashConfig(filename string) (string, error) { - content, err := ReadConfig(filename) - if err != nil { - return "", err - } - hash, err := LocalitySensitiveHash(AsUnifiedLines(content)) - return hash, err -} - -func Distance(left, right string) (int, error) { - if len(left) != 70 || len(left) != len(right) { - return 999999, errors.New("Incorrect length of TLSH hashes.") - } - leftish, err := tlsh.ParseStringToTlsh(left) - if err != nil { - return 0, err - } - rightish, err := tlsh.ParseStringToTlsh(right) - if err != nil { - return 0, err - } - return leftish.Diff(rightish), nil -} diff --git a/conda/config_test.go b/conda/config_test.go index 33aa1e08..9e6db36d 100644 --- a/conda/config_test.go +++ b/conda/config_test.go @@ -24,7 +24,7 @@ func TestReadingCorrectFileProducesText(t *testing.T) { text, err := conda.ReadConfig("testdata/conda.yaml") must_be.Nil(err) wont_be.Text("", text) - must_be.Equal(167, len(text)) + must_be.Equal(169, len(text)) } func TestUnifyLineWorksCorrectly(t *testing.T) { @@ -54,82 +54,3 @@ func TestGetsLinesAsUnifiedCorrectly(t *testing.T) { must_be.Equal([]string{"a"}, conda.AsUnifiedLines("a\r\n\r\n")) must_be.Equal([]string{"a", "b"}, conda.AsUnifiedLines(" \r\n\tb \r\na\r\n\ta\t\r\n")) } - -func TestCanCalculateHash(t *testing.T) { - must_be, wont_be := hamlet.Specifications(t) - - expected := "8a900000303c000003000003030000000000033cc000000c030cf00000c03000000000" - actual, err := conda.LocalitySensitiveHash([]string{"a", "b", "c"}) - must_be.Nil(err) - wont_be.Nil(actual) - must_be.Equal(expected, actual) -} - -func TestCanCalculateHashEvenOnEmptySet(t *testing.T) { - must_be, wont_be := hamlet.Specifications(t) - - expected := "aa900000000c000000000000000000000000000c000000000000000000003000000000" - actual, err := conda.LocalitySensitiveHash([]string{}) - must_be.Nil(err) - wont_be.Nil(actual) - must_be.Equal(expected, actual) -} - -func TestCanCalculateHashEvenOnEmptyString(t *testing.T) { - must_be, wont_be := hamlet.Specifications(t) - - expected := "aa900000000c000000000000000000000000000c000000000000000000003000000000" - actual, err := conda.LocalitySensitiveHash([]string{""}) - must_be.Nil(err) - wont_be.Nil(actual) - must_be.Equal(expected, actual) -} - -func TestCanCalculateHashForConfig(t *testing.T) { - must_be, wont_be := hamlet.Specifications(t) - - actual, err := conda.HashConfig("missing/bad.yaml") - wont_be.Nil(err) - must_be.Equal("", actual) - - expected := "ded08c86224cc710b22228d3a1aa1a074bdf1a44f01be819c0a816044eebb80242030a" - actual, err = conda.HashConfig("testdata/conda.yaml") - must_be.Nil(err) - must_be.Equal(expected, actual) - - other := "59c02b47324cc310a3332cc3a19a160b4bef0a04f02ff415c0f410044ddb780342030a" - actual, err = conda.HashConfig("testdata/other.yaml") - must_be.Nil(err) - must_be.Equal(other, actual) - - third := "a8d08c86224cc710b22228c3a1aa1a0b4bef1a44f01fa815c0a412044aaa780242030a" - actual, err = conda.HashConfig("testdata/third.yaml") - must_be.Nil(err) - must_be.Equal(third, actual) - - distance, err := conda.Distance(expected, other) - must_be.Nil(err) - must_be.Equal(94, distance) - - distance, err = conda.Distance(expected, third) - must_be.Nil(err) - must_be.Equal(13, distance) - - alien := "8a900000303c000003000003030000000000033cc000000c030cf00000c03000000000" - - distance, err = conda.Distance(expected, alien) - must_be.Nil(err) - must_be.Equal(384, distance) - - distance, err = conda.Distance("", alien) - wont_be.Nil(err) - must_be.Equal(999999, distance) - - distance, err = conda.Distance("iiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiii", alien) - wont_be.Nil(err) - must_be.Equal(0, distance) - - distance, err = conda.Distance(alien, "iiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiii") - wont_be.Nil(err) - must_be.Equal(0, distance) -} diff --git a/conda/dependencies.go b/conda/dependencies.go new file mode 100644 index 00000000..6bd78631 --- /dev/null +++ b/conda/dependencies.go @@ -0,0 +1,218 @@ +package conda + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "text/tabwriter" + + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/fail" + "github.com/robocorp/rcc/pathlib" + "github.com/robocorp/rcc/pretty" + "gopkg.in/yaml.v2" +) + +type dependency struct { + Name string `yaml:"name" json:"name"` + Version string `yaml:"version" json:"version"` + Origin string `yaml:"origin" json:"channel"` +} + +func (it *dependency) AsKey() string { + return fmt.Sprintf("%-50s %-20s", it.Name, it.Origin) +} + +type dependencies []*dependency + +func (it dependencies) sorted() dependencies { + sort.SliceStable(it, func(left, right int) bool { + lefty := strings.ToLower(it[left].Name) + righty := strings.ToLower(it[right].Name) + if lefty == righty { + return it[left].Origin < it[right].Origin + } + return lefty < righty + }) + return it +} + +func (it dependencies) WarnVulnerability(url, severity, name string, versions ...string) { + found, ok := it.Lookup(name, false) + if !ok { + found, ok = it.Lookup(name, true) + } + if !ok { + return + } + for _, version := range versions { + if found.Version == version { + pretty.Highlight("Dependency with %s severity vulnerability detected: %s %s. For more information see %s", severity, name, found.Version, url) + } + } +} + +func (it dependencies) Lookup(name string, pypi bool) (*dependency, bool) { + for _, entry := range it { + if pypi && entry.Origin != "pypi" { + continue + } + if !pypi && entry.Origin == "pypi" { + continue + } + if entry.Name == name { + return entry, true + } + } + return nil, false +} + +func parseDependencies(origin string, output []byte) (dependencies, error) { + result := make(dependencies, 0, 100) + err := json.Unmarshal(output, &result) + if err != nil { + return nil, err + } + if len(origin) == 0 { + return result, nil + } + for _, here := range result { + if len(here.Origin) == 0 { + here.Origin = origin + } + } + return result, nil +} + +func fillDependencies(context, targetFolder string, seen map[string]string, collector dependencies, command ...string) (_ dependencies, err error) { + defer fail.Around(&err) + + task, err := livePrepare(targetFolder, command...) + fail.On(err != nil, "%v", err) + out, _, err := task.CaptureOutput() + fail.On(err != nil, "%v", err) + listing, err := parseDependencies(context, []byte(out)) + fail.On(err != nil, "%v", err) + for _, entry := range listing { + found, ok := seen[strings.ToLower(entry.Name)] + if ok && found == entry.Version { + continue + } + collector = append(collector, entry) + seen[strings.ToLower(entry.Name)] = entry.Version + } + return collector, nil +} + +func GoldenMasterFilename(targetFolder string) string { + return filepath.Join(targetFolder, "golden-ee.yaml") +} + +func goldenMaster(targetFolder string, pipUsed bool) (err error) { + defer fail.Around(&err) + + seen := make(map[string]string) + collector := make(dependencies, 0, 100) + collector, err = fillDependencies("mamba", targetFolder, seen, collector, BinMicromamba(), "list", "--json") + fail.On(err != nil, "Failed to list micromamba dependencies, reason: %v", err) + if pipUsed { + collector, err = fillDependencies("pypi", targetFolder, seen, collector, "pip", "list", "--isolated", "--local", "--format", "json") + fail.On(err != nil, "Failed to list pip dependencies, reason: %v", err) + } + body, err := yaml.Marshal(collector.sorted()) + fail.On(err != nil, "Failed to make yaml, reason: %v", err) + goldenfile := GoldenMasterFilename(targetFolder) + common.Debug("%sGolden EE file at: %v%s", pretty.Yellow, goldenfile, pretty.Reset) + return pathlib.WriteFile(goldenfile, body, 0644) +} + +func LoadWantedDependencies(filename string) dependencies { + body, err := os.ReadFile(filename) + if err != nil { + return dependencies{} + } + result := make(dependencies, 0, 100) + err = yaml.Unmarshal(body, &result) + if err != nil { + return dependencies{} + } + return result.sorted() +} + +func SideBySideViewOfDependencies(goldenfile, wantedfile string) (err error) { + defer fail.Around(&err) + + gold := LoadWantedDependencies(goldenfile) + want := LoadWantedDependencies(wantedfile) + + if len(gold) == 0 && len(want) == 0 { + return fmt.Errorf("Running against old environment, and no dependencies.yaml.") + } + + diffmap := make(map[string][2]int) + injectDiffmap(diffmap, want, 0) + injectDiffmap(diffmap, gold, 1) + keyset := make([]string, 0, len(diffmap)) + for key, _ := range diffmap { + keyset = append(keyset, key) + } + sort.Strings(keyset) + + common.WaitLogs() + hasgold := false + unknown := fmt.Sprintf("%sUnknown%s", pretty.Grey, pretty.Reset) + same := fmt.Sprintf("%sSame%s", pretty.Cyan, pretty.Reset) + drifted := fmt.Sprintf("%sDrifted%s", pretty.Yellow, pretty.Reset) + missing := fmt.Sprintf("%sN/A%s", pretty.Grey, pretty.Reset) + tabbed := tabwriter.NewWriter(os.Stderr, 2, 4, 2, ' ', 0) + tabbed.Write([]byte("Wanted\tVersion\tOrigin\t|\tNo.\t|\tAvailable\tVersion\tOrigin\t|\tStatus\n")) + tabbed.Write([]byte("------\t-------\t------\t+\t---\t+\t---------\t-------\t------\t+\t------\n")) + for at, key := range keyset { + left, right, status := "-\t-\t-\t", "\t-\t-\t-", unknown + sides := diffmap[key] + if sides[0] < 0 || sides[1] < 0 { + status = missing + } else { + left, right := want[sides[0]], gold[sides[1]] + if left.Version == right.Version { + status = same + } else { + status = drifted + } + } + if sides[0] > -1 { + entry := want[sides[0]] + left = fmt.Sprintf("%s\t%s\t%s\t", entry.Name, entry.Version, entry.Origin) + } + if sides[1] > -1 { + entry := gold[sides[1]] + right = fmt.Sprintf("\t%s\t%s\t%s", entry.Name, entry.Version, entry.Origin) + hasgold = true + } + data := fmt.Sprintf("%s|\t%3d\t|%s\t|\t%s\n", left, at+1, right, status) + tabbed.Write([]byte(data)) + } + tabbed.Write([]byte("------\t-------\t------\t+\t---\t+\t---------\t-------\t------\t+\t------\n")) + tabbed.Write([]byte("Wanted\tVersion\tOrigin\t|\tNo.\t|\tAvailable\tVersion\tOrigin\t|\tStatus\n")) + tabbed.Write([]byte("\n")) + tabbed.Flush() + if !hasgold { + return fmt.Errorf("Running against old environment, which does not have 'golden-ee.yaml' file.") + } + return nil +} + +func injectDiffmap(diffmap map[string][2]int, deps dependencies, side int) { + for at, entry := range deps { + key := entry.AsKey() + found, ok := diffmap[key] + if !ok { + found = [2]int{-1, -1} + } + found[side] = at + diffmap[key] = found + } +} diff --git a/conda/diagnosis.go b/conda/diagnosis.go new file mode 100644 index 00000000..f1682d70 --- /dev/null +++ b/conda/diagnosis.go @@ -0,0 +1,106 @@ +package conda + +import ( + "fmt" + "path/filepath" + "sort" + "strings" + + "github.com/robocorp/rcc/common" +) + +func MakeRelativeMap(root string, entries map[string]string) map[string]string { + result := make(map[string]string) + for key, value := range entries { + if !strings.HasPrefix(key, root) { + result[key] = value + continue + } + short, err := filepath.Rel(root, key) + if err == nil { + key = short + } + result[key] = value + } + return result +} + +func DirhashDiff(history, future map[string]string, warning bool) { + removed := []string{} + added := []string{} + changed := []string{} + for key, value := range history { + next, ok := future[key] + if !ok { + removed = append(removed, key) + continue + } + if value != next { + changed = append(changed, key) + } + } + for key, _ := range future { + _, ok := history[key] + if !ok { + added = append(added, key) + } + } + if len(removed)+len(added)+len(changed) == 0 { + return + } + common.Log("---- rcc env diff ----") + sort.Strings(removed) + sort.Strings(added) + sort.Strings(changed) + separate := false + for _, folder := range removed { + common.Trace("- diff: removed %q", folder) + separate = true + } + if len(changed) > 0 { + if separate { + common.Trace("-------") + separate = false + } + for _, folder := range changed { + common.Trace("- diff: changed %q", folder) + separate = true + } + } + if len(added) > 0 { + if separate { + common.Trace("-------") + separate = false + } + for _, folder := range added { + common.Trace("- diff: added %q", folder) + separate = true + } + } + if warning { + if separate { + common.Trace("-------") + separate = false + } + common.Log("Notice: Robot run modified the environment which will slow down the next run.") + common.Log(" Please inform the robot developer about this. Use --trace for details.") + } + common.Log("---- rcc env diff ----") +} + +func DiagnoseDirty(beforeLabel, afterLabel string, beforeHash, afterHash []byte, beforeErr, afterErr error, beforeDetails, afterDetails map[string]string, warning bool) { + if beforeErr != nil || afterErr != nil { + common.Debug("live %q diagnosis failed, before: %v, after: %v", afterLabel, beforeErr, afterErr) + return + } + beforeSummary := fmt.Sprintf("%02x", beforeHash) + afterSummary := fmt.Sprintf("%02x", afterHash) + if beforeSummary == afterSummary { + common.Debug("live %q diagnosis: did not change during run [%s]", afterLabel, afterSummary) + return + } + common.Debug("live %q diagnosis: corrupted [%s] => [%s]", afterLabel, beforeSummary, afterSummary) + beforeDetails = MakeRelativeMap(beforeLabel, beforeDetails) + afterDetails = MakeRelativeMap(afterLabel, afterDetails) + DirhashDiff(beforeDetails, afterDetails, warning) +} diff --git a/conda/download.go b/conda/download.go deleted file mode 100644 index e6fe272d..00000000 --- a/conda/download.go +++ /dev/null @@ -1,38 +0,0 @@ -package conda - -import ( - "crypto/sha256" - "io" - "net/http" - "os" - - "github.com/robocorp/rcc/common" -) - -func DownloadConda() error { - url := DownloadLink() - filename := DownloadTarget() - response, err := http.Get(url) - if err != nil { - return err - } - defer response.Body.Close() - - out, err := os.Create(filename) - if err != nil { - return err - } - defer out.Close() - - digest := sha256.New() - many := io.MultiWriter(out, digest) - - common.Debug("Downloading %s <%s> -> %s", url, response.Status, filename) - - _, err = io.Copy(many, response.Body) - if err != nil { - return err - } - - return common.Debug("SHA256 sum: %02x", digest.Sum(nil)) -} diff --git a/conda/environment_test.go b/conda/environment_test.go index e8123eae..26204dc0 100644 --- a/conda/environment_test.go +++ b/conda/environment_test.go @@ -10,11 +10,5 @@ import ( func TestHasDownloadLinkAvailable(t *testing.T) { must_be, _ := hamlet.Specifications(t) - must_be.True(len(conda.DownloadLink()) > 10) -} - -func TestCanCreateDownloadTarget(t *testing.T) { - must_be, _ := hamlet.Specifications(t) - - must_be.True(len(conda.DownloadTarget()) > 10) + must_be.True(len(conda.MicromambaLink()) > 10) } diff --git a/conda/extarnallymanaged.go b/conda/extarnallymanaged.go new file mode 100644 index 00000000..5ea1822c --- /dev/null +++ b/conda/extarnallymanaged.go @@ -0,0 +1,55 @@ +package conda + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + + "github.com/robocorp/rcc/blobs" + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/fail" +) + +const ( + EXTERNALLY_MANAGED = "EXTERNALLY-MANAGED" +) + +type ( + SysconfigPaths struct { + Stdlib string `json:"stdlib"` + Purelib string `json:"purelib"` + Platlib string `json:"platlib"` + } +) + +func FindSysconfigPaths(path string) (paths *SysconfigPaths, err error) { + defer fail.Around(&err) + + capture, code, err := LiveCapture(path, "python", "-c", "import json, sysconfig; print(json.dumps(sysconfig.get_paths()))") + if err != nil { + common.Fatal(fmt.Sprintf("EXTERNALLY-MANAGED failure [%d/%x]", code, code), err) + return nil, err + } + mappings := &SysconfigPaths{} + err = json.Unmarshal([]byte(capture), mappings) + fail.Fast(err) + return mappings, nil +} + +func ApplyExternallyManaged(path string) (label string, err error) { + defer fail.Around(&err) + + if !common.ExternallyManaged { + return "", nil + } + common.Debug("Applying EXTERNALLY-MANAGED (PEP 668) to environment.") + paths, err := FindSysconfigPaths(path) + fail.Fast(err) + location := filepath.Join(paths.Stdlib, EXTERNALLY_MANAGED) + blob, err := blobs.Asset("assets/externally_managed.txt") + fail.Fast(err) + fail.Fast(os.WriteFile(location, blob, 0o644)) + common.Timeline("applied EXTERNALLY-MANAGED (PEP 668) to this holotree space") + return fmt.Sprintf("%s (PEP 668) ", EXTERNALLY_MANAGED), nil +} diff --git a/conda/installing.go b/conda/installing.go index 53342e44..16ac9613 100644 --- a/conda/installing.go +++ b/conda/installing.go @@ -1,48 +1,68 @@ package conda import ( + "bytes" + "compress/gzip" + "io" + "os" + "time" + + "github.com/robocorp/rcc/blobs" + "github.com/robocorp/rcc/cloud" "github.com/robocorp/rcc/common" - "github.com/robocorp/rcc/shell" + "github.com/robocorp/rcc/fail" + "github.com/robocorp/rcc/pathlib" + "github.com/robocorp/rcc/pretty" ) -func MustConda() bool { - return HasConda() || (ValidateLocations() && (DoDownload() || DoDownload() || DoDownload()) && DoInstall()) +func MustMicromamba() bool { + return HasMicroMamba() || DoExtract(1*time.Millisecond) || DoExtract(1*time.Second) || DoExtract(3*time.Second) || DoFailMicromamba() } -func DoDownload() bool { - if common.DebugFlag { - defer common.Stopwatch("Download done in").Report() - } +func DoFailMicromamba() bool { + pretty.Exit(113, "Could not extract micromamba, see above stream for more details.") + return false +} - common.Log("Downloading Miniconda, this may take awhile ...") +func GunzipWrite(context, filename string, blob []byte) (err error) { + defer fail.Around(&err) - err := DownloadConda() - if err != nil { - common.Error("Download", err) - return false - } else { - common.Log("Verify checksum from https://docs.conda.io/en/latest/miniconda.html") - return true - } + stream := bytes.NewReader(blob) + source, err := gzip.NewReader(stream) + fail.On(err != nil, "Failed to %q -> %v", filename, err) + + sink, err := pathlib.Create(filename) + fail.On(err != nil, "Failed to create %q reader -> %v", context, err) + defer sink.Close() + + _, err = io.Copy(sink, source) + fail.On(err != nil, "Failed to copy %q to %q -> %v", context, filename, err) + + err = sink.Sync() + fail.On(err != nil, "Failed to sync %q -> %v", filename, err) + + return nil } -func DoInstall() bool { - if common.DebugFlag { - defer common.Stopwatch("Installation done in").Report() - } +func DoExtract(delay time.Duration) bool { + pretty.Highlight("Note: Extracting micromamba binary from inside rcc.") - if !ValidateLocations() { + time.Sleep(delay) + binary := blobs.MustMicromamba() + err := GunzipWrite("micromamba", BinMicromamba(), binary) + if err != nil { + err = os.Remove(BinMicromamba()) + if err != nil { + common.Fatal("Remove of micromamba failed, reason:", err) + } return false } - - common.Log("Installing Miniconda, this may take awhile ...") - - install := InstallCommand() - common.Debug("Running: %v", install) - _, err := shell.New(nil, ".", install...).Transparent() + err = os.Chmod(BinMicromamba(), 0o755) if err != nil { - common.Error("Install", err) + common.Fatal("Could not make micromamba executalbe, reason:", err) return false } + cloud.InternalBackgroundMetric(common.ControllerIdentity(), "rcc.micromamba.extract", common.Version) + common.PlatformSyncDelay() return true } diff --git a/conda/librarian.go b/conda/librarian.go deleted file mode 100644 index 44769851..00000000 --- a/conda/librarian.go +++ /dev/null @@ -1,109 +0,0 @@ -package conda - -type Changes struct { - Name string - Dryrun bool - Pip bool - Channel bool - Add []string - Remove []string -} - -func UpdateEnvironment(filename string, changes *Changes) (string, error) { - environment := SummonEnvironment(filename) - if changes.Channel { - updateChannels(environment, changes) - } else { - err := updatePackages(environment, changes) - if err != nil { - return "", err - } - } - if len(changes.Name) > 0 { - environment.Name = changes.Name - } - if changes.Dryrun { - return environment.AsYaml() - } - err := environment.SaveAs(filename) - if err != nil { - return "", err - } - return environment.AsYaml() -} - -func Index(search string, members []string) int { - for at, member := range members { - if member == search { - return at - } - } - return -1 -} - -func updateChannels(environment *Environment, changes *Changes) { - result := make([]string, 0, len(changes.Add)+len(environment.Channels)) - for _, current := range environment.Channels { - if Index(current, changes.Remove) > -1 { - continue - } - result = append(result, current) - } - for _, here := range changes.Add { - if Index(here, result) > -1 { - continue - } - result = append(result, here) - } - environment.Channels = result -} - -func updatePackages(environment *Environment, changes *Changes) error { - adds := asDependencies(changes.Add) - removes := asDependencies(changes.Remove) - if changes.Pip { - result, err := composePackages(environment.Pip, adds, removes) - if err != nil { - return err - } - environment.Pip = result - } else { - result, err := composePackages(environment.Conda, adds, removes) - if err != nil { - return err - } - environment.Conda = result - } - return nil -} - -func composePackages(target []*Dependency, add []*Dependency, remove []*Dependency) ([]*Dependency, error) { - result := make([]*Dependency, 0, len(target)+len(add)) - for _, current := range target { - if current.Index(remove) > -1 { - continue - } - result = append(result, current) - } - for _, current := range add { - found := current.Index(result) - if found < 0 { - result = append(result, current) - continue - } - selected, err := current.ChooseSpecific(result[found]) - if err != nil { - return nil, err - } - result[found] = selected - } - return result, nil -} - -func asDependencies(labels []string) []*Dependency { - result := make([]*Dependency, 0, len(labels)) - for _, label := range labels { - result = append(result, AsDependency(label)) - } - return result -} diff --git a/conda/packageyaml.go b/conda/packageyaml.go new file mode 100644 index 00000000..5a44a95d --- /dev/null +++ b/conda/packageyaml.go @@ -0,0 +1,103 @@ +package conda + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/robocorp/rcc/cloud" + "github.com/robocorp/rcc/pathlib" + "gopkg.in/yaml.v2" +) + +type ( + packageDependencies struct { + CondaForge []string `yaml:"conda-forge,omitempty"` + Pypi []string `yaml:"pypi,omitempty"` + } + internalPackage struct { + Dependencies *packageDependencies `yaml:"dependencies"` + PostInstall []string `yaml:"post-install,omitempty"` + } +) + +func (it *internalPackage) AsEnvironment() *Environment { + result := &Environment{ + Channels: []string{"conda-forge"}, + PostInstall: []string{}, + } + seenScripts := make(map[string]bool) + result.PostInstall = addItem(seenScripts, it.PostInstall, result.PostInstall) + pushConda(result, it.condaDependencies()) + pushPip(result, it.pipDependencies()) + result.pipPromote() + return result +} + +func fixPipDependency(dependency *Dependency) *Dependency { + if dependency != nil { + if dependency.Qualifier == "=" { + dependency.Original = fmt.Sprintf("%s==%s", dependency.Name, dependency.Versions) + dependency.Qualifier = "==" + } + } + return dependency +} + +func (it *internalPackage) pipDependencies() []*Dependency { + result := make([]*Dependency, 0, len(it.Dependencies.Pypi)) + for _, item := range it.Dependencies.Pypi { + dependency := AsDependency(item) + if dependency != nil { + result = append(result, fixPipDependency(dependency)) + } + } + return result +} + +func (it *internalPackage) condaDependencies() []*Dependency { + result := make([]*Dependency, 0, len(it.Dependencies.CondaForge)) + for _, item := range it.Dependencies.CondaForge { + dependency := AsDependency(item) + if dependency != nil { + result = append(result, dependency) + } + } + return result +} + +func packageYamlFrom(content []byte) (*Environment, error) { + result := new(internalPackage) + err := yaml.Unmarshal(content, result) + if err != nil { + return nil, err + } + return result.AsEnvironment(), nil +} + +func ReadPackageCondaYaml(filename string) (*Environment, error) { + basename := strings.ToLower(filepath.Base(filename)) + if basename == "package.yaml" { + environment, err := ReadPackageYaml(filename) + if err == nil { + return environment, nil + } + } + return readCondaYaml(filename) +} + +func ReadPackageYaml(filename string) (*Environment, error) { + var content []byte + var err error + + if pathlib.IsFile(filename) { + content, err = os.ReadFile(filename) + } else { + content, err = cloud.ReadFile(filename) + } + if err != nil { + return nil, fmt.Errorf("%q: %w", filename, err) + } + return packageYamlFrom(content) +} diff --git a/conda/plananalyzer.go b/conda/plananalyzer.go new file mode 100644 index 00000000..abcba10b --- /dev/null +++ b/conda/plananalyzer.go @@ -0,0 +1,173 @@ +package conda + +import ( + "bytes" + "fmt" + "regexp" + "strings" + "time" + + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/pretty" +) + +const ( + newline = '\n' + spacing = "\r\n\t " +) + +var ( + planPattern = regexp.MustCompile("^--- (.+?) plan @\\d+.\\d+s ---$") + pipNotePrefixes = [][2]string{ + {"info:", "%s [plan analyzer]"}, + {"warning:", "%s [plan analyzer]"}, + {"error:", "%s [plan analyzer]"}, + {"successfully uninstalled", "%s [plan analyzer: pip overrides conda]"}, + {"building wheels", "%s [plan analyzer: missing pip wheel files]"}, + } + pipNoteContains = [][2]string{ + {"failed to build", "%s [plan analyzer: build failure]"}, + } + pipDetailContains = [][2]string{ + {"which is incompatible", "%s [plan analyzer: pip vs. conda?]"}, + } +) + +type ( + AnalyzerStrategy func(*PlanAnalyzer, string) + StrategyMap map[string]AnalyzerStrategy + RepeatCache map[string]bool + + PlanAnalyzer struct { + Strategies StrategyMap + Active AnalyzerStrategy + Notes []string + Pending []byte + Repeats RepeatCache + Realtime bool + Details bool + Started time.Time + } +) + +func NewPlanAnalyzer(realtime bool) *PlanAnalyzer { + strategies := make(StrategyMap) + strategies["micromamba"] = ignoreStrategy + strategies["post install"] = ignoreStrategy + strategies["activation"] = ignoreStrategy + strategies["pip check"] = ignoreStrategy + strategies["pip"] = pipStrategy + return &PlanAnalyzer{ + Strategies: strategies, + Active: ignoreStrategy, + Notes: []string{}, + Pending: nil, + Repeats: make(RepeatCache), + Realtime: realtime, + Details: false, + } +} + +func pipStrategy(ref *PlanAnalyzer, event string) { + low := strings.TrimSpace(strings.ToLower(event)) + note := "" + detail := "" + for _, marker := range pipNotePrefixes { + if strings.HasPrefix(low, marker[0]) { + note = fmt.Sprintf(marker[1], event) + } + } + for _, marker := range pipNoteContains { + if strings.Contains(low, marker[0]) { + note = fmt.Sprintf(marker[1], event) + } + } + if strings.Contains(low, "using cached") { + if strings.Contains(low, ".tar.gz") { + detail = fmt.Sprintf("%s [plan analyzer: missing wheel file?]", event) + } else { + detail = fmt.Sprintf("%s [plan analyzer]", event) + } + } + for _, marker := range pipDetailContains { + if strings.Contains(low, marker[0]) { + detail = fmt.Sprintf(marker[1], event) + } + } + elapsed := time.Since(ref.Started).Round(1 * time.Second) + if len(note) > 0 { + ref.Notes = append(ref.Notes, note) + if ref.Realtime { + pretty.Warning("%s @%s", strings.TrimSpace(note), elapsed) + } + ref.Details = true + return + } + if ref.Details && len(detail) > 0 { + ref.Notes = append(ref.Notes, detail) + if ref.Realtime { + pretty.Note("%s @%s", detail, elapsed) + } + return + } + if ref.Realtime { + common.Trace("PIP: %s", event) + } +} + +func ignoreStrategy(ref *PlanAnalyzer, event string) { + // does nothing by default +} + +func (it *PlanAnalyzer) Observe(event string) { + found := planPattern.FindStringSubmatch(event) + if len(found) > 1 { + it.Active = ignoreStrategy + strategy, ok := it.Strategies[found[1]] + if ok { + it.Active = strategy + } + it.Repeats = make(RepeatCache) + it.Details = false + it.Started = time.Now() + } + it.Active(it, event) +} + +func (it *PlanAnalyzer) Write(blob []byte) (int, error) { + old := len(it.Pending) + update := len(blob) + var total uint64 = uint64(old) + uint64(update) + body := make([]byte, 0, total) + if old > 0 { + body = append(body, it.Pending...) + } + if update > 0 { + body = append(body, blob...) + } + terminator := []byte{newline} + parts := bytes.SplitAfter(body, terminator) + size := len(parts) + last := parts[size-1] + terminated := bytes.HasSuffix(last, terminator) + if !terminated { + it.Pending = last + parts = parts[:size-1] + } else { + it.Pending = nil + } + for _, part := range parts { + it.Observe(strings.TrimRight(string(part), spacing)) + } + return update, nil +} + +func (it *PlanAnalyzer) Close() { + if len(it.Notes) == 0 || it.Realtime { + return + } + pretty.Warning("Analyzing installation plan revealed following findings:") + for _, note := range it.Notes { + common.Log(" %s* %s%s%s", pretty.Cyan, pretty.Bold, strings.TrimSpace(note), pretty.Reset) + } +} diff --git a/conda/platform_darwin.go b/conda/platform_darwin.go new file mode 100644 index 00000000..661eac76 --- /dev/null +++ b/conda/platform_darwin.go @@ -0,0 +1,70 @@ +package conda + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/robocorp/rcc/blobs" + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/pathlib" + "github.com/robocorp/rcc/pretty" + "github.com/robocorp/rcc/settings" +) + +const ( + Newline = "\n" + binSuffix = "/bin" + activateScript = `#!/bin/bash + +export MAMBA_ROOT_PREFIX={{.MambaRootPrefix}} +eval "$('{{.Micromamba}}' shell activate -s bash -p {{.Live}})" +"{{.Rcc}}" internal env -l after +` + commandSuffix = ".sh" +) + +var ( + Shell = []string{"bash", "--noprofile", "--norc", "-i"} + FileExtensions = []string{"", ".sh"} +) + +func CondaEnvironment() []string { + env := os.Environ() + env = append(env, fmt.Sprintf("MAMBA_ROOT_PREFIX=%s", common.MambaRootPrefix())) + if !common.DisableTempManagement() { + tempFolder := common.ProductTemp() + env = append(env, fmt.Sprintf("TEMP=%s", tempFolder)) + env = append(env, fmt.Sprintf("TMP=%s", tempFolder)) + } + return injectNetworkEnvironment(env) +} + +func BinMicromamba() string { + location := common.ExpandPath(filepath.Join(common.MicromambaLocation(), blobs.MicromambaVersion())) + err := pathlib.EnsureDirectoryExists(location) + if err != nil { + pretty.Warning("Problem creating %q, reason: %v", location, err) + } + return common.ExpandPath(filepath.Join(location, "micromamba")) +} + +func CondaPaths(prefix string) []string { + return []string{prefix + binSuffix} +} + +func MicromambaLink() string { + return settings.Global.DownloadsLink(micromambaLink("macos64", "micromamba")) +} + +func IsWindows() bool { + return false +} + +func HasLongPathSupport() bool { + return true +} + +func EnforceLongpathSupport() error { + return nil +} diff --git a/conda/platform_darwin_amd64.go b/conda/platform_darwin_amd64.go deleted file mode 100644 index 2df4321c..00000000 --- a/conda/platform_darwin_amd64.go +++ /dev/null @@ -1,62 +0,0 @@ -package conda - -import ( - "os" - "path/filepath" -) - -const ( - Newline = "\n" - defaultRobocorpLocation = "$HOME/.robocorp" - binSuffix = "/bin" -) - -var ( - Shell = []string{"bash", "--noprofile", "--norc", "-i"} - FileExtensions = []string{"", ".sh"} -) - -func ExpandPath(entry string) string { - intermediate := os.ExpandEnv(entry) - result, err := filepath.Abs(intermediate) - if err != nil { - return intermediate - } - return result -} - -func BinConda() string { - return ExpandPath(filepath.Join(MinicondaLocation(), "bin", "conda")) -} - -func BinPython() string { - return ExpandPath(filepath.Join(MinicondaLocation(), "bin", "python")) -} - -func CondaPaths(prefix string) []string { - return []string{prefix + binSuffix} -} - -func DownloadLink() string { - return "https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh" -} - -func DownloadTarget() string { - return filepath.Join(os.TempDir(), "miniconda3.sh") -} - -func InstallCommand() []string { - return []string{"bash", DownloadTarget(), "-u", "-b", "-p", MinicondaLocation()} -} - -func IsPosix() bool { - return true -} - -func IsWindows() bool { - return false -} - -func ValidateLocations() bool { - return true -} diff --git a/conda/platform_linux.go b/conda/platform_linux.go index 8448e805..0b6b7599 100644 --- a/conda/platform_linux.go +++ b/conda/platform_linux.go @@ -1,14 +1,27 @@ package conda import ( + "fmt" "os" "path/filepath" + + "github.com/robocorp/rcc/blobs" + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/pathlib" + "github.com/robocorp/rcc/pretty" + "github.com/robocorp/rcc/settings" ) const ( - Newline = "\n" - defaultRobocorpLocation = "$HOME/.robocorp" - binSuffix = "/bin" + Newline = "\n" + binSuffix = "/bin" + activateScript = `#!/bin/bash + +export MAMBA_ROOT_PREFIX={{.MambaRootPrefix}} +eval "$('{{.Micromamba}}' shell activate -s bash -p {{.Live}})" +"{{.Rcc}}" internal env -l after +` + commandSuffix = ".sh" ) var ( @@ -16,39 +29,42 @@ var ( Shell = []string{"bash", "--noprofile", "--norc", "-i"} ) -func ExpandPath(entry string) string { - intermediate := os.ExpandEnv(entry) - result, err := filepath.Abs(intermediate) - if err != nil { - return intermediate - } - return result +func MicromambaLink() string { + return settings.Global.DownloadsLink(micromambaLink("linux64", "micromamba")) } -func BinConda() string { - return ExpandPath(filepath.Join(MinicondaLocation(), "bin", "conda")) +func CondaEnvironment() []string { + env := os.Environ() + env = append(env, fmt.Sprintf("MAMBA_ROOT_PREFIX=%s", common.MambaRootPrefix())) + if !common.DisableTempManagement() { + tempFolder := common.ProductTemp() + env = append(env, fmt.Sprintf("TEMP=%s", tempFolder)) + env = append(env, fmt.Sprintf("TMP=%s", tempFolder)) + } + return injectNetworkEnvironment(env) } -func BinPython() string { - return ExpandPath(filepath.Join(MinicondaLocation(), "bin", "python")) +func BinMicromamba() string { + location := common.ExpandPath(filepath.Join(common.MicromambaLocation(), blobs.MicromambaVersion())) + err := pathlib.EnsureDirectoryExists(location) + if err != nil { + pretty.Warning("Problem creating %q, reason: %v", location, err) + } + return common.ExpandPath(filepath.Join(location, "micromamba")) } func CondaPaths(prefix string) []string { - return []string{ExpandPath(prefix + binSuffix)} -} - -func InstallCommand() []string { - return []string{"bash", DownloadTarget(), "-u", "-b", "-p", MinicondaLocation()} -} - -func IsPosix() bool { - return true + return []string{common.ExpandPath(prefix + binSuffix)} } func IsWindows() bool { return false } -func ValidateLocations() bool { +func HasLongPathSupport() bool { return true } + +func EnforceLongpathSupport() error { + return nil +} diff --git a/conda/platform_linux_386.go b/conda/platform_linux_386.go deleted file mode 100644 index f8159f92..00000000 --- a/conda/platform_linux_386.go +++ /dev/null @@ -1,14 +0,0 @@ -package conda - -import ( - "os" - "path/filepath" -) - -func DownloadLink() string { - return "https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86.sh" -} - -func DownloadTarget() string { - return filepath.Join(os.TempDir(), "miniconda3.sh") -} diff --git a/conda/platform_linux_amd64.go b/conda/platform_linux_amd64.go deleted file mode 100644 index aeedc371..00000000 --- a/conda/platform_linux_amd64.go +++ /dev/null @@ -1,14 +0,0 @@ -package conda - -import ( - "os" - "path/filepath" -) - -func DownloadLink() string { - return "https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh" -} - -func DownloadTarget() string { - return filepath.Join(os.TempDir(), "miniconda3.sh") -} diff --git a/conda/platform_test.go b/conda/platform_test.go index 59efdead..2dcbb343 100644 --- a/conda/platform_test.go +++ b/conda/platform_test.go @@ -3,6 +3,7 @@ package conda_test import ( "testing" + "github.com/robocorp/rcc/common" "github.com/robocorp/rcc/conda" "github.com/robocorp/rcc/hamlet" ) @@ -13,7 +14,7 @@ func TestExpandingPath(t *testing.T) { } _, wont_be := hamlet.Specifications(t) - wont_be.Equal("$HOME/bin", conda.ExpandPath("$HOME/bin")) + wont_be.Equal("$HOME/bin", common.ExpandPath("$HOME/bin")) } func TestCondaPathSetup(t *testing.T) { @@ -31,8 +32,7 @@ func TestFlagsAreCorrectlySet(t *testing.T) { if conda.IsWindows() { t.Skip("Not a windows test.") } - must_be, wont_be := hamlet.Specifications(t) + _, wont_be := hamlet.Specifications(t) wont_be.True(conda.IsWindows()) - must_be.True(conda.IsPosix()) } diff --git a/conda/platform_windows.go b/conda/platform_windows.go index 0f03a9be..b01516b4 100644 --- a/conda/platform_windows.go +++ b/conda/platform_windows.go @@ -1,54 +1,61 @@ package conda import ( + "fmt" "os" "path/filepath" - "regexp" + + "golang.org/x/sys/windows/registry" + + "github.com/robocorp/rcc/blobs" + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/pathlib" + "github.com/robocorp/rcc/pretty" + "github.com/robocorp/rcc/settings" + "github.com/robocorp/rcc/shell" ) const ( - Newline = "\r\n" - defaultRobocorpLocation = "%LOCALAPPDATA%\\robocorp" - librarySuffix = "\\Library" - scriptSuffix = "\\Scripts" - usrSuffix = "\\bin" - binSuffix = "\\bin" + mingwSuffix = "\\mingw-w64" + Newline = "\r\n" + librarySuffix = "\\Library" + scriptSuffix = "\\Scripts" + usrSuffix = "\\usr" + binSuffix = "\\bin" + activateScript = "@echo off\n" + + "set \"MAMBA_ROOT_PREFIX={{.MambaRootPrefix}}\"\n" + + "for /f \"tokens=* usebackq\" %%a in ( `call \"{{.Micromamba}}\" shell activate -s cmd.exe -p \"{{.Live}}\"` ) do ( call \"%%a\" )\n" + + "call \"{{.Rcc}}\" internal env -l after\n" + commandSuffix = ".cmd" ) +func MicromambaLink() string { + return settings.Global.DownloadsLink(micromambaLink("windows64", "micromamba.exe")) +} + var ( - Shell = []string{"cmd.exe", "/K"} - variablePattern = regexp.MustCompile("%[a-zA-Z]+%") - FileExtensions = []string{".exe", ".com", ".bat", ".cmd", ""} + Shell = []string{"cmd.exe", "/K"} + FileExtensions = []string{".exe", ".com", ".bat", ".cmd", ""} ) -func fromEnvironment(form string) string { - replacement, ok := os.LookupEnv(form[1 : len(form)-1]) - if ok { - return replacement - } - replacement, ok = os.LookupEnv(form) - if ok { - return replacement +func CondaEnvironment() []string { + env := os.Environ() + env = append(env, fmt.Sprintf("MAMBA_ROOT_PREFIX=%s", common.MambaRootPrefix())) + if !common.DisableTempManagement() { + tempFolder := common.ProductTemp() + env = append(env, fmt.Sprintf("TEMP=%s", tempFolder)) + env = append(env, fmt.Sprintf("TMP=%s", tempFolder)) } - return form + return injectNetworkEnvironment(env) } -func ExpandPath(entry string) string { - intermediate := os.ExpandEnv(entry) - intermediate = variablePattern.ReplaceAllStringFunc(intermediate, fromEnvironment) - result, err := filepath.Abs(intermediate) +func BinMicromamba() string { + location := common.ExpandPath(filepath.Join(common.MicromambaLocation(), blobs.MicromambaVersion())) + err := pathlib.EnsureDirectoryExists(location) if err != nil { - return intermediate + pretty.Warning("Problem creating %q, reason: %v", location, err) } - return result -} - -func BinConda() string { - return ExpandPath(filepath.Join(MinicondaLocation(), "Scripts", "conda.exe")) -} - -func BinPython() string { - return ExpandPath(filepath.Join(MinicondaLocation(), "python.exe")) + return common.ExpandPath(filepath.Join(location, "micromamba.exe")) } func CondaPaths(prefix string) []string { @@ -62,24 +69,36 @@ func CondaPaths(prefix string) []string { } } -func InstallCommand() []string { - return []string{DownloadTarget(), "/InstallationType=JustMe", "/NoRegisty=1", "/S", "/D=" + MinicondaLocation()} +func IsWindows() bool { + return true } -func IsPosix() bool { - return false -} +func HasLongPathSupport() bool { + baseline := []string{common.Product.Home(), fmt.Sprintf("stump%x", os.Getpid())} + stumpath := filepath.Join(baseline...) + defer os.RemoveAll(stumpath) -func IsWindows() bool { + for count := 0; count < 24; count++ { + baseline = append(baseline, fmt.Sprintf("verylongpath%d", count+1)) + } + fullpath := filepath.Join(baseline...) + + code, err := shell.New(nil, ".", "cmd.exe", "/c", "mkdir", fullpath).StderrOnly().Transparent() + common.Trace("Checking long path support with MKDIR '%v' (%d characters) -> %v [%v] {%d}", fullpath, len(fullpath), err == nil, err, code) + if err != nil { + longPathSupportArticle := settings.Global.DocsLink("troubleshooting/windows-long-path") + common.Log("%sWARNING! Long path support failed. Reason: %v.%s", pretty.Red, err, pretty.Reset) + common.Log("%sWARNING! See %v for more details.%s", pretty.Red, longPathSupportArticle, pretty.Reset) + return false + } return true } -func ValidateLocations() bool { - checked := map[string]string{ - "Environment variable 'TMP'": os.Getenv("TMP"), - "Environment variable 'TEMP'": os.Getenv("TEMP"), - "Environment variable 'ROBOCORP_HOME'": os.Getenv("ROBOCORP_HOME"), - "Path to 'ROBOCORP_HOME' directory": RobocorpHome(), +func EnforceLongpathSupport() error { + key, _, err := registry.CreateKey(registry.LOCAL_MACHINE, `SYSTEM\CurrentControlSet\Control\FileSystem`, registry.SET_VALUE) + if err != nil { + return err } - return validateLocations(checked) + defer key.Close() + return key.SetDWordValue("LongPathsEnabled", 1) } diff --git a/conda/platform_windows_386.go b/conda/platform_windows_386.go deleted file mode 100644 index bd1440b6..00000000 --- a/conda/platform_windows_386.go +++ /dev/null @@ -1,18 +0,0 @@ -package conda - -import ( - "os" - "path/filepath" -) - -const ( - mingwSuffix = "\\mingw-w32" -) - -func DownloadLink() string { - return "https://repo.anaconda.com/miniconda/Miniconda3-latest-Windows-x86.exe" -} - -func DownloadTarget() string { - return filepath.Join(os.TempDir(), "miniconda3.exe") -} diff --git a/conda/platform_windows_amd64.go b/conda/platform_windows_amd64.go deleted file mode 100644 index 4a2d6f93..00000000 --- a/conda/platform_windows_amd64.go +++ /dev/null @@ -1,18 +0,0 @@ -package conda - -import ( - "os" - "path/filepath" -) - -const ( - mingwSuffix = "\\mingw-w64" -) - -func DownloadLink() string { - return "https://repo.anaconda.com/miniconda/Miniconda3-latest-Windows-x86_64.exe" -} - -func DownloadTarget() string { - return filepath.Join(os.TempDir(), "miniconda3.exe") -} diff --git a/conda/robocorp.go b/conda/robocorp.go index 98e493ed..6d374eff 100644 --- a/conda/robocorp.go +++ b/conda/robocorp.go @@ -5,28 +5,51 @@ import ( "fmt" "os" "path/filepath" + "regexp" "sort" + "strconv" + "strings" + "github.com/robocorp/rcc/blobs" "github.com/robocorp/rcc/common" "github.com/robocorp/rcc/pathlib" -) - -const ( - ROBOCORP_HOME_VARIABLE = `ROBOCORP_HOME` + "github.com/robocorp/rcc/pretty" + "github.com/robocorp/rcc/settings" + "github.com/robocorp/rcc/shell" + "github.com/robocorp/rcc/xviper" ) var ( - ignoredPaths = []string{"python", "conda"} - pythonPaths = []string{"resources", "libraries", "tasks", "variables"} + ignoredPaths = []string{ + "python", + "conda", + "pyenv", + "venv", + "pypoetry", + ".poetry", + "virtualenv", + } + hashPattern = regexp.MustCompile("^[0-9a-f]{16}(?:\\.meta)?$") + versionPattern = regexp.MustCompile("^[^0-9]*([0-9.]+).*$") ) +func micromambaLink(platform, filename string) string { + return fmt.Sprintf("micromamba/%s/%s/%s", blobs.MicromambaVersion(), platform, filename) +} + func sorted(files []os.FileInfo) { sort.SliceStable(files, func(left, right int) bool { return files[left].Name() < files[right].Name() }) } -func DigestFor(folder string) ([]byte, error) { +func ignoreDynamicDirectories(folder, entryName string) bool { + base := strings.ToLower(filepath.Base(folder)) + name := strings.ToLower(entryName) + return name == "__pycache__" || (name == "gen" && base == "comtypes") +} + +func DigestFor(folder string, collect map[string]string) ([]byte, error) { handle, err := os.Open(folder) if err != nil { return nil, err @@ -40,10 +63,10 @@ func DigestFor(folder string) ([]byte, error) { sorted(entries) for _, entry := range entries { if entry.IsDir() { - if entry.Name() == "__pycache__" { + if ignoreDynamicDirectories(folder, entry.Name()) { continue } - digest, err := DigestFor(filepath.Join(folder, entry.Name())) + digest, err := DigestFor(filepath.Join(folder, entry.Name()), collect) if err != nil { return nil, err } @@ -54,36 +77,15 @@ func DigestFor(folder string) ([]byte, error) { digester.Write([]byte(repr)) } result := digester.Sum([]byte{}) + if collect != nil { + key := fmt.Sprintf("%02x", result) + collect[folder] = key + } return result, nil } -func hasMetafile(basedir, subdir string) bool { - folder := filepath.Join(basedir, subdir) - _, err := os.Stat(metafile(folder)) - return err == nil -} - -func dirnamesFrom(location string) []string { - result := make([]string, 0, 20) - handle, err := os.Open(ExpandPath(location)) - if err != nil { - common.Error("Warning", err) - return result - } - defer handle.Close() - children, err := handle.Readdir(-1) - if err != nil { - common.Error("Warning", err) - return result - } - - for _, child := range children { - if child.IsDir() && hasMetafile(location, child.Name()) { - result = append(result, child.Name()) - } - } - - return result +func HolotreePath(environment string) pathlib.PathParts { + return pathlib.PathFrom(CondaPaths(environment)...) } func FindPath(environment string) pathlib.PathParts { @@ -93,97 +95,207 @@ func FindPath(environment string) pathlib.PathParts { return target } -func PythonPath() pathlib.PathParts { - return pathlib.PathFrom(pythonPaths...) +func FindPython(location string) (string, bool) { + holotreePath := HolotreePath(location) + python, ok := holotreePath.Which("python3", FileExtensions) + if ok { + return python, ok + } + return holotreePath.Which("python", FileExtensions) +} + +func FindUv(location string) (string, bool) { + holotreePath := HolotreePath(location) + uv, ok := holotreePath.Which("uv", FileExtensions) + if ok { + return uv, ok + } + return holotreePath.Which("uv", FileExtensions) +} + +func injectNetworkEnvironment(environment []string) []string { + if settings.Global.NoRevocation() { + environment = append(environment, "MAMBA_SSL_NO_REVOKE=true") + } + if !settings.Global.VerifySsl() { + environment = append(environment, "MAMBA_SSL_VERIFY=false") + environment = append(environment, "RC_DISABLE_SSL=true") + environment = append(environment, "WDM_SSL_VERIFY=0") + environment = append(environment, "NODE_TLS_REJECT_UNAUTHORIZED=0") + } + if settings.Global.LegacyRenegotiation() { + environment = append(environment, "RC_TLS_LEGACY_RENEGOTIATION_ALLOWED=true") + } + environment = appendIfValue(environment, "https_proxy", settings.Global.HttpsProxy()) + environment = appendIfValue(environment, "HTTPS_PROXY", settings.Global.HttpsProxy()) + environment = appendIfValue(environment, "http_proxy", settings.Global.HttpProxy()) + environment = appendIfValue(environment, "HTTP_PROXY", settings.Global.HttpProxy()) + environment = appendIfValue(environment, "no_proxy", settings.Global.NoProxy()) + environment = appendIfValue(environment, "NO_PROXY", settings.Global.NoProxy()) + if common.WarrantyVoided() { + environment = append(environment, "RCC_WARRANTY_VOIDED=true") + } + return environment +} + +func removeIncompatibleEnvironmentVariables(environment []string, unwanted ...string) []string { + result := make([]string, 0, len(environment)) +search: + for _, here := range environment { + parts := strings.Split(strings.TrimSpace(here), "=") + for _, name := range unwanted { + if strings.EqualFold(name, parts[0]) { + pretty.Warning("Removing incompatible variable %q from environment.", here) + continue search + } + } + result = append(result, here) + } + return result } -func EnvironmentExtensionFor(location string) []string { - environment := make([]string, 0, 20) - searchPath := FindPath(location) - python, ok := searchPath.Which("python3", FileExtensions) +func CondaExecutionEnvironment(location string, inject []string, full bool) []string { + environment := make([]string, 0, 100) + if full { + environment = append(environment, os.Environ()...) + environment = removeIncompatibleEnvironmentVariables(environment, "VIRTUAL_ENV") + } + if inject != nil && len(inject) > 0 { + environment = append(environment, inject...) + } + holotreePath := HolotreePath(location) + python, ok := holotreePath.Which("python3", FileExtensions) if !ok { - python, ok = searchPath.Which("python", FileExtensions) + python, ok = holotreePath.Which("python", FileExtensions) } if ok { environment = append(environment, "PYTHON_EXE="+python) } - return append(environment, + if !common.DisablePycManagement() { + environment = append(environment, + "PYTHONDONTWRITEBYTECODE=x", + "PYTHONPYCACHEPREFIX="+common.ProductTemp(), + ) + } else { + common.Timeline(".pyc file management was disabled.") + } + if !common.DisableTempManagement() { + environment = append(environment, + "TEMP="+common.ProductTemp(), + "TMP="+common.ProductTemp(), + ) + } else { + common.Timeline("temp directory management was disabled.") + } + environment = append(environment, "CONDA_DEFAULT_ENV=rcc", - "CONDA_EXE="+BinConda(), "CONDA_PREFIX="+location, - "CONDA_PROMPT_MODIFIER=(rcc)", - "CONDA_PYTHON_EXE="+BinPython(), + "CONDA_PROMPT_MODIFIER=(rcc) ", "CONDA_SHLVL=1", "PYTHONHOME=", "PYTHONSTARTUP=", "PYTHONEXECUTABLE=", "PYTHONNOUSERSITE=1", - "ROBOCORP_HOME="+RobocorpHome(), - searchPath.AsEnvironmental("PATH"), - PythonPath().AsEnvironmental("PYTHONPATH"), + fmt.Sprintf("%s=%s", common.Product.HomeVariable(), common.Product.Home()), + "RCC_ENVIRONMENT_HASH="+common.EnvironmentHash, + "RCC_INSTALLATION_ID="+xviper.TrackingIdentity(), + "RCC_HOLOTREE_SPACE_ROOT="+location, + "RCC_TRACKING_ALLOWED="+fmt.Sprintf("%v", xviper.CanTrack()), + "RCC_EXE="+common.BinRcc(), + "RCC_VERSION="+common.Version, + FindPath(location).AsEnvironmental("PATH"), ) -} - -func EnvironmentFor(location string) []string { - return append(os.Environ(), EnvironmentExtensionFor(location)...) -} - -func CondaExecutable() string { - return ExpandPath(filepath.Join(MinicondaLocation(), "condabin", "conda")) -} - -func CondaCache() string { - return ExpandPath(filepath.Join(MinicondaLocation(), "pkgs", "cache")) -} - -func HasConda() bool { - location := ExpandPath(filepath.Join(MinicondaLocation(), "condabin")) - stat, err := os.Stat(location) - if err == nil && stat.IsDir() { - return true + environment = append(environment, LoadActivationEnvironment(location)...) + environment = injectNetworkEnvironment(environment) + if settings.Global.HasPipRc() { + environment = appendIfValue(environment, "PIP_CONFIG_FILE", common.PipRcFile()) } - return false -} - -func RobocorpHome() string { - home := os.Getenv(ROBOCORP_HOME_VARIABLE) - if len(home) > 0 { - return ExpandPath(home) + if settings.Global.HasCaBundle() { + environment = appendIfValue(environment, "REQUESTS_CA_BUNDLE", common.CaBundleFile()) + environment = appendIfValue(environment, "CURL_CA_BUNDLE", common.CaBundleFile()) + environment = appendIfValue(environment, "SSL_CERT_FILE", common.CaBundleFile()) + environment = appendIfValue(environment, "NODE_EXTRA_CA_CERTS", common.CaBundleFile()) } - return ExpandPath(defaultRobocorpLocation) + return environment } -func LiveLocation() string { - return filepath.Join(RobocorpHome(), "live") -} - -func TemplateLocation() string { - return filepath.Join(RobocorpHome(), "base") +func appendIfValue(environment []string, key, value string) []string { + if len(value) > 0 { + return append(environment, key+"="+value) + } + return environment } -func MinicondaLocation() string { - return filepath.Join(RobocorpHome(), "miniconda3") +func AsVersion(incoming string) (uint64, string) { + incoming = strings.TrimSpace(incoming) + versionText := "0" +search: + for _, line := range strings.SplitN(incoming, "\n", -1) { + found := versionPattern.FindStringSubmatch(line) + if found != nil { + versionText = found[1] + break search + } + } + parts := strings.SplitN(versionText, ".", 4) + steps := len(parts) + multipliers := []uint64{1000000, 1000, 1} + version := uint64(0) + for at, multiplier := range multipliers { + if steps <= at { + break + } + value, err := strconv.ParseUint(parts[at], 10, 64) + if err != nil { + break + } + version += multiplier * value + } + return version, versionText } -func ensureDirectory(name string) string { - pathlib.EnsureDirectoryExists(name) - return name +func UvVersion(uv string) string { + environment := CondaExecutionEnvironment(".", nil, true) + versionText, _, err := shell.New(environment, ".", uv, "--version").CaptureOutput() + if err != nil { + return err.Error() + } + _, versionText = AsVersion(versionText) + return versionText } -func PipCache() string { - return ensureDirectory(filepath.Join(RobocorpHome(), "pipcache")) +func PipVersion(python string) string { + environment := CondaExecutionEnvironment(".", nil, true) + versionText, _, err := shell.New(environment, ".", python, "-m", "pip", "--version").CaptureOutput() + if err != nil { + return err.Error() + } + _, versionText = AsVersion(versionText) + return versionText } -func WheelCache() string { - return ensureDirectory(filepath.Join(RobocorpHome(), "wheels")) +func MicromambaVersion() string { + versionText, _, err := shell.New(CondaEnvironment(), ".", BinMicromamba(), "--repodata-ttl", "90000", "--version").CaptureOutput() + if err != nil { + return err.Error() + } + _, versionText = AsVersion(versionText) + return versionText } -func RobotCache() string { - return ensureDirectory(filepath.Join(RobocorpHome(), "robots")) +func HasMicroMamba() bool { + if !pathlib.IsFile(BinMicromamba()) { + return false + } + version, versionText := AsVersion(MicromambaVersion()) + goodEnough := version >= blobs.MicromambaVersionLimit + common.Debug("%q version is %q -> %v (good enough: %v)", BinMicromamba(), versionText, version, goodEnough) + common.Timeline("Âĩmamba version is %q (at %q).", versionText, BinMicromamba()) + return goodEnough } func LocalChannel() (string, bool) { - basefolder := filepath.Join(RobocorpHome(), "channel") + basefolder := filepath.Join(common.Product.Home(), "channel") fullpath := filepath.Join(basefolder, "channeldata.json") stats, err := os.Stat(fullpath) if err != nil { @@ -194,19 +306,3 @@ func LocalChannel() (string, bool) { } return "", false } - -func TemplateFrom(hash string) string { - return filepath.Join(TemplateLocation(), hash) -} - -func LiveFrom(hash string) string { - return ExpandPath(filepath.Join(LiveLocation(), hash)) -} - -func TemplateList() []string { - return dirnamesFrom(TemplateLocation()) -} - -func LiveList() []string { - return dirnamesFrom(LiveLocation()) -} diff --git a/conda/robocorp_test.go b/conda/robocorp_test.go new file mode 100644 index 00000000..cd5618fa --- /dev/null +++ b/conda/robocorp_test.go @@ -0,0 +1,37 @@ +package conda_test + +import ( + "testing" + + "github.com/robocorp/rcc/blobs" + "github.com/robocorp/rcc/conda" + "github.com/robocorp/rcc/hamlet" +) + +func second(_ interface{}, version string) string { + return version +} + +func TestCanParseMicromambaVersion(t *testing.T) { + must_be, _ := hamlet.Specifications(t) + + must_be.Equal("0", second(conda.AsVersion("python"))) + must_be.Equal("0.19.1", second(conda.AsVersion("0.19.1"))) + must_be.Equal("0.19.0", second(conda.AsVersion("micromamba: 0.19.0"))) + must_be.Equal("0.19.0", second(conda.AsVersion("\n\n\tmicromamba: 0.19.0 \nlibmamba: 0.18.7\n\n\t"))) + must_be.Equal("0.20", second(conda.AsVersion("microrumba: 0.20"))) +} + +func TestCanParsePipVersion(t *testing.T) { + must_be, _ := hamlet.Specifications(t) + + must_be.Equal("20.3.4", second(conda.AsVersion("pip 20.3.4 from /outer/space/python/blah (python 3.9)"))) + must_be.Equal("22.2.2", second(conda.AsVersion("pip 22.2.2 from /outer/space/python/blah (python 3.9)"))) +} + +func TestInternalMicromambaVersionConsistency(t *testing.T) { + must_be, _ := hamlet.Specifications(t) + + needs, _ := conda.AsVersion(blobs.MicromambaVersion()) + must_be.Equal(uint64(blobs.MicromambaVersionLimit), needs) +} diff --git a/conda/testdata/conda.yaml b/conda/testdata/conda.yaml index a259ca18..cbc425cc 100644 --- a/conda/testdata/conda.yaml +++ b/conda/testdata/conda.yaml @@ -1,10 +1,10 @@ channels: - - defaults - conda-forge + - defaults dependencies: - - python=3.7.5 + - python=3.9.13 - pip - robotframework=3.1 - robotframework-seleniumlibrary - pip: - - webdrivermanager \ No newline at end of file + - webdrivermanager diff --git a/conda/testdata/layers.yaml b/conda/testdata/layers.yaml new file mode 100644 index 00000000..cbee87c6 --- /dev/null +++ b/conda/testdata/layers.yaml @@ -0,0 +1,9 @@ +channels: +- conda-forge +dependencies: +- python=3.9.13 +- pip=22.1.2 +- pip: + - rpaframework==22.5.3 +rccPostInstall: +- python3 -m pip --version diff --git a/conda/testdata/other.yaml b/conda/testdata/other.yaml index 99bcdf68..69943fb8 100644 --- a/conda/testdata/other.yaml +++ b/conda/testdata/other.yaml @@ -1,7 +1,7 @@ channels: - defaults dependencies: - - python=3.7.5 + - python=3.9.13 - pip - robotframework=3.2 - robotframework-seleniumlibrary diff --git a/conda/testdata/third.yaml b/conda/testdata/third.yaml index 19ec48ce..6721b269 100644 --- a/conda/testdata/third.yaml +++ b/conda/testdata/third.yaml @@ -1,8 +1,8 @@ channels: - - defaults - conda-forge + - defaults dependencies: - - python=3.7.5 + - python=3.9.13 - pip - robotframework=3.2 - robotframework-seleniumlibrary diff --git a/conda/validate.go b/conda/validate.go index 592f0aa8..914a6329 100644 --- a/conda/validate.go +++ b/conda/validate.go @@ -1,34 +1,46 @@ package conda import ( + "fmt" "regexp" - "strings" "github.com/robocorp/rcc/common" "github.com/robocorp/rcc/pretty" ) var ( - validPathCharacters = regexp.MustCompile("(?i)^[.a-z0-9_:/\\\\]+$") + validPathCharacters = regexp.MustCompile("(?i)^[.a-z0-9_:/\\\\~-]+$") ) +func ValidLocation(value string) bool { + return validPathCharacters.MatchString(value) +} + func validateLocations(checked map[string]string) bool { success := true for name, value := range checked { if len(value) == 0 { continue } - if strings.ContainsAny(value, " \t") { - success = false - common.Log("%sWARNING! %s contain spaces. Cannot install miniconda at %v.%s", pretty.Red, name, value, pretty.Reset) - } - if !validPathCharacters.MatchString(value) { + if !ValidLocation(value) { success = false - common.Log("%sWARNING! %s contain illegal characters. Cannot install miniconda at %v.%s", pretty.Red, name, value, pretty.Reset) + common.Log("%sWARNING! %s contain illegal characters. Cannot use tooling with path %q.%s", pretty.Yellow, name, value, pretty.Reset) } } if !success { - common.Log("%sERROR! Cannot install miniconda on your system. See above.%s", pretty.Red, pretty.Reset) + common.Log("%sWARNING! Python pip might not work correctly in your system. See above.%s", pretty.Yellow, pretty.Reset) } return success } + +func ValidateLocations() bool { + checked := map[string]string{ + //"Environment variable 'TMP'": os.Getenv("TMP"), + //"Environment variable 'TEMP'": os.Getenv("TEMP"), + fmt.Sprintf("Path to '%s' directory", common.Product.HomeVariable()): common.Product.Home(), + } + // 7.1.2021 -- just warnings for now -- JMP:FIXME:JMP later + validateLocations(checked) + return true + // return validateLocations(checked) +} diff --git a/conda/workflows.go b/conda/workflows.go index 57e7d836..70e3fb74 100644 --- a/conda/workflows.go +++ b/conda/workflows.go @@ -3,356 +3,560 @@ package conda import ( "errors" "fmt" - "io/ioutil" + "io" + "math/rand" "os" "path/filepath" + "strings" "time" + "github.com/robocorp/rcc/cloud" "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/journal" "github.com/robocorp/rcc/pathlib" + "github.com/robocorp/rcc/pretty" + "github.com/robocorp/rcc/settings" "github.com/robocorp/rcc/shell" - "github.com/robocorp/rcc/xviper" ) -func chooseBestEnvironment(best int, selected, reference string, candidates []string) (int, string) { - for _, candidate := range candidates { - move, err := Distance(reference, candidate) - if err != nil { - continue - } - if move < best { - best, selected = move, candidate - } - } +const ( + SkipNoLayers SkipLayer = iota + SkipMicromambaLayer SkipLayer = iota + SkipPipLayer SkipLayer = iota + SkipPostinstallLayer SkipLayer = iota + SkipError SkipLayer = iota +) - return best, selected -} +const ( + micromambaInstall = `micromamba install` + pipInstall = `pip install` + uvInstall = `uv install` + postInstallScripts = `post-install script execution` +) -func Hexdigest(raw []byte) string { - return fmt.Sprintf("%02x", raw) -} +type ( + pipTool func(string, string, string, fmt.Stringer, io.Writer) (bool, bool, bool, string) -func metafile(folder string) string { - return ExpandPath(folder + ".meta") -} + SkipLayer uint8 + Recorder interface { + Record([]byte) error + } + PlanWriter struct { + filename string + blob []byte + } +) -func metaLoad(location string) (string, error) { - raw, err := ioutil.ReadFile(metafile(location)) - if err != nil { - return "", err +func NewPlanWriter(filename string) *PlanWriter { + return &PlanWriter{ + filename: filename, + blob: make([]byte, 0, 50000), } - return string(raw), nil } -func metaSave(location, data string) error { - return ioutil.WriteFile(metafile(location), []byte(data), 0644) +func (it *PlanWriter) AsText() string { + return string(it.blob) } -func touchMetafile(location string) { - pathlib.TouchWhen(metafile(location), time.Now()) +func (it *PlanWriter) Write(blob []byte) (int, error) { + it.blob = append(it.blob, blob...) + return len(blob), nil } -func LastUsed(location string) (time.Time, error) { - return pathlib.Modtime(metafile(location)) +func (it *PlanWriter) Save() error { + return os.WriteFile(it.filename, it.blob, 0o644) } -func IsPristine(folder string) bool { - digest, err := DigestFor(folder) - if err != nil { - return false - } - meta, err := metaLoad(folder) - if err != nil { - return false - } - return Hexdigest(digest) == meta -} - -func reuseExistingLive(key string) bool { - candidate := LiveFrom(key) - if IsPristine(candidate) { - touchMetafile(candidate) - return true - } - removeClone(candidate) - return false +func metafile(folder string) string { + return common.ExpandPath(folder + ".meta") } -func LiveExecution(liveFolder string, command ...string) error { - searchPath := FindPath(liveFolder) +func livePrepare(liveFolder string, command ...string) (*shell.Task, error) { commandName := command[0] - task, ok := searchPath.Which(commandName, FileExtensions) + task, ok := HolotreePath(liveFolder).Which(commandName, FileExtensions) if !ok { - return errors.New(fmt.Sprintf("Cannot find command: %v", commandName)) + return nil, fmt.Errorf("Cannot find command: %v", commandName) } common.Debug("Using %v as command %v.", task, commandName) command[0] = task - environment := EnvironmentFor(liveFolder) - _, err := shell.New(environment, ".", command...).Transparent() - return err + environment := CondaExecutionEnvironment(liveFolder, nil, true) + return shell.New(environment, ".", command...), nil } -func newLive(condaYaml, requirementsText, key string, force, freshInstall bool) bool { - targetFolder := LiveFrom(key) - when := time.Now() - if force { - when = when.Add(-20 * 24 * time.Hour) - } - if force || !freshInstall { - common.Log("rcc touching conda cache. (Stamp: %v)", when) - SilentTouch(CondaCache(), when) - } - common.Debug("Setting up new conda environment using %v to folder %v", condaYaml, targetFolder) - command := []string{CondaExecutable(), "env", "create", "-q", "-f", condaYaml, "-p", targetFolder} - if common.DebugFlag { - command = []string{CondaExecutable(), "env", "create", "-f", condaYaml, "-p", targetFolder} +func LiveCapture(liveFolder string, command ...string) (string, int, error) { + task, err := livePrepare(liveFolder, command...) + if err != nil { + return "", 9999, err } - _, err := shell.New(nil, ".", command...).Transparent() + return task.CaptureOutput() +} + +func LiveExecution(sink io.Writer, liveFolder string, command ...string) (int, error) { + fmt.Fprintf(sink, "Command %q at %q:\n", command, liveFolder) + task, err := livePrepare(liveFolder, command...) if err != nil { - common.Error("Conda error", err) - return false + return 0, err } - common.Debug("Updating new environment at %v with pip requirements from %v", targetFolder, requirementsText) - pipCommand := []string{"pip", "install", "--no-color", "--disable-pip-version-check", "--prefer-binary", "--cache-dir", PipCache(), "--find-links", WheelCache(), "--requirement", requirementsText, "--quiet"} - if common.DebugFlag { - pipCommand = []string{"pip", "install", "--no-color", "--disable-pip-version-check", "--prefer-binary", "--cache-dir", PipCache(), "--find-links", WheelCache(), "--requirement", requirementsText} + return task.Tracked(sink, false) +} + +type InstallObserver map[string]bool + +func (it InstallObserver) Write(content []byte) (int, error) { + text := strings.ToLower(string(content)) + if strings.Contains(text, "safetyerror:") { + it["safetyerror"] = true } - err = LiveExecution(targetFolder, pipCommand...) - if err != nil { - common.Error("Pip error", err) - return false + if strings.Contains(text, "pkgs") { + it["pkgs"] = true } - digest, err := DigestFor(targetFolder) - if err != nil { - common.Error("Digest", err) - return false + if strings.Contains(text, "appears to be corrupted") { + it["corrupted"] = true } - return metaSave(targetFolder, Hexdigest(digest)) == nil + return len(content), nil } -func temporaryConfig(condaYaml, requirementsText string, filenames ...string) (string, error) { - var left, right *Environment - var err error +func (it InstallObserver) HasFailures(targetFolder string) bool { + if it["safetyerror"] && it["corrupted"] && len(it) > 2 { + cloud.InternalBackgroundMetric(common.ControllerIdentity(), "rcc.env.creation.failure", common.Version) + renameRemove(targetFolder) + location := filepath.Join(common.Product.Home(), "pkgs") + common.Log("%sWARNING! Conda environment is unstable, see above error.%s", pretty.Red, pretty.Reset) + common.Log("%sWARNING! To fix it, try to remove directory: %v%s", pretty.Red, location, pretty.Reset) + return true + } + return false +} - for _, filename := range filenames { - left = right - right, err = ReadCondaYaml(filename) +func newLive(yaml, condaYaml, requirementsText, key string, force, freshInstall bool, skip SkipLayer, finalEnv *Environment, recorder Recorder) (bool, error) { + if !MustMicromamba() { + return false, fmt.Errorf("Could not get micromamba installed.") + } + targetFolder := common.StageFolder + if skip == SkipNoLayers { + common.Debug("=== pre cleanup phase ===") + common.Timeline("pre cleanup phase.") + err := renameRemove(targetFolder) if err != nil { - return "", err + return false, err } - if left == nil { - continue - } - right, err = left.Merge(right) + } + common.Debug("=== first try phase ===") + common.Timeline("first try.") + success, fatal := newLiveInternal(yaml, condaYaml, requirementsText, key, force, freshInstall, skip, finalEnv, recorder) + if !success && !force && !fatal && !common.NoRetryBuild { + journal.CurrentBuildEvent().Rebuild() + cloud.InternalBackgroundMetric(common.ControllerIdentity(), "rcc.env.creation.retry", common.Version) + common.Debug("=== second try phase ===") + common.Timeline("second try.") + common.ForceDebug() + common.Log("Retry! First try failed ... now retrying with debug and force options!") + err := renameRemove(targetFolder) if err != nil { - return "", err + return false, err } + success, _ = newLiveInternal(yaml, condaYaml, requirementsText, key, true, freshInstall, SkipNoLayers, finalEnv, recorder) } - yaml, err := right.AsYaml() - if err != nil { - return "", err - } - hash, err := LocalitySensitiveHash(AsUnifiedLines(yaml)) - if err != nil { - return "", err - } - err = right.SaveAsRequirements(requirementsText) - if err != nil { - return "", err + if success { + journal.CurrentBuildEvent().Successful() } - pure := right.AsPureConda() - return hash, pure.SaveAs(condaYaml) + return success, nil } -func NewEnvironment(force bool, configurations ...string) (string, error) { - requests := xviper.GetInt("stats.env.request") + 1 - hits := xviper.GetInt("stats.env.hit") - dirty := xviper.GetInt("stats.env.dirty") - misses := xviper.GetInt("stats.env.miss") - failures := xviper.GetInt("stats.env.failures") - merges := xviper.GetInt("stats.env.merges") - templates := len(TemplateList()) - freshInstall := templates == 0 +func assertStageFolder(location string) { + base := filepath.Base(location) + holotree := strings.HasPrefix(base, "h") && strings.HasSuffix(base, "t") + virtual := strings.HasPrefix(base, "v") && strings.HasSuffix(base, "h") + if !(holotree || virtual) { + panic(fmt.Sprintf("FATAL: incorrect stage %q for environment building!", location)) + } +} - defer func() { - common.Log("#### Progress: 4/4 [Done.] [Stats: %d environments, %d requests, %d merges, %d hits, %d dirty, %d misses, %d failures | %s]", templates, requests, merges, hits, dirty, misses, failures, common.Version) - }() - common.Log("#### Progress: 0/4 [try use existing live same environment?] %v", xviper.TrackingIdentity()) +func micromambaLayer(fingerprint, condaYaml, targetFolder string, stopwatch fmt.Stringer, planWriter io.Writer, force bool) (bool, bool) { + assertStageFolder(targetFolder) + common.TimelineBegin("Layer: micromamba [%s]", fingerprint) + defer common.TimelineEnd() - xviper.Set("stats.env.request", requests) + common.Debug("Setting up new conda environment using %v to folder %v", condaYaml, targetFolder) + ttl := "57600" + if force { + ttl = "0" + } + pretty.Progress(7, "Running micromamba phase. (micromamba v%s) [layer: %s]", MicromambaVersion(), fingerprint) + mambaCommand := common.NewCommander(BinMicromamba(), "create", "--always-copy", "--no-env", "--safety-checks", "enabled", "--extra-safety-checks", "--retry-clean-cache", "--strict-channel-priority", "--repodata-ttl", ttl, "-y", "-f", condaYaml, "-p", targetFolder) + mambaCommand.Option("--channel-alias", settings.Global.CondaURL()) + mambaCommand.ConditionalFlag(common.VerboseEnvironmentBuilding(), "--verbose") + mambaCommand.ConditionalFlag(!settings.Global.HasMicroMambaRc(), "--no-rc") + mambaCommand.ConditionalFlag(settings.Global.HasMicroMambaRc(), "--rc-file", common.MicroMambaRcFile()) + observer := make(InstallObserver) + common.Debug("=== micromamba create phase ===") + fmt.Fprintf(planWriter, "\n--- micromamba plan @%ss ---\n\n", stopwatch) + tee := io.MultiWriter(observer, planWriter) + code, err := shell.New(CondaEnvironment(), ".", mambaCommand.CLI()...).Tracked(tee, false) + if err != nil || code != 0 { + cloud.InternalBackgroundMetric(common.ControllerIdentity(), "rcc.env.fatal.micromamba", fmt.Sprintf("%d_%x", code, code)) + common.Timeline("micromamba fail.") + common.Fatal(fmt.Sprintf("Micromamba [%d/%x]", code, code), err) + pretty.RccPointOfView(micromambaInstall, err) + return false, false + } + journal.CurrentBuildEvent().MicromambaComplete() + common.Timeline("micromamba done.") + if observer.HasFailures(targetFolder) { + return false, true + } + return true, false +} - if len(configurations) > 1 { - merges += 1 - xviper.Set("stats.env.merges", merges) +func uvLayer(fingerprint, requirementsText, targetFolder string, stopwatch fmt.Stringer, planWriter io.Writer) (bool, bool, bool, string) { + assertStageFolder(targetFolder) + common.TimelineBegin("Layer: uv [%s]", fingerprint) + defer common.TimelineEnd() + + pipUsed := false + fmt.Fprintf(planWriter, "\n--- uv plan @%ss ---\n\n", stopwatch) + uv, uvok := FindUv(targetFolder) + if !uvok { + fmt.Fprintf(planWriter, "Note: no uv in target folder: %s\n", targetFolder) + return false, false, pipUsed, "" + } + python, pyok := FindPython(targetFolder) + if !pyok { + fmt.Fprintf(planWriter, "Note: no python in target folder: %s\n", targetFolder) + } + uvCache, wheelCache := common.UvCache(), common.WheelCache() + size, ok := pathlib.Size(requirementsText) + if !ok || size == 0 { + pretty.Progress(8, "Skipping pip install phase -- no pip dependencies.") + } else { + pretty.Progress(8, "Running uv install phase. (uv v%s) [layer: %s]", UvVersion(uv), fingerprint) + common.Debug("Updating new environment at %v with uv requirements from %v (size: %v)", targetFolder, requirementsText, size) + uvCommand := common.NewCommander(uv, "pip", "install", "--link-mode", "copy", "--color", "never", "--cache-dir", uvCache, "--find-links", wheelCache, "--requirement", requirementsText) + uvCommand.Option("--index-url", settings.Global.PypiURL()) + // no "--trusted-host" on uv pip install + // uvCommand.Option("--trusted-host", settings.Global.PypiTrustedHost()) + uvCommand.ConditionalFlag(common.VerboseEnvironmentBuilding(), "--verbose") + common.Debug("=== uv install phase ===") + code, err := LiveExecution(planWriter, targetFolder, uvCommand.CLI()...) + if err != nil || code != 0 { + cloud.InternalBackgroundMetric(common.ControllerIdentity(), "rcc.env.fatal.uv", fmt.Sprintf("%d_%x", code, code)) + common.Timeline("uv fail.") + common.Fatal(fmt.Sprintf("uv [%d/%x]", code, code), err) + pretty.RccPointOfView(uvInstall, err) + return false, false, pipUsed, "" + } + journal.CurrentBuildEvent().PipComplete() + common.Timeline("uv done.") + pipUsed = true } + return true, false, pipUsed, python +} - marker := time.Now().Unix() - condaYaml := filepath.Join(os.TempDir(), fmt.Sprintf("conda_%x.yaml", marker)) - requirementsText := filepath.Join(os.TempDir(), fmt.Sprintf("require_%x.txt", marker)) - common.Debug("Using temporary conda.yaml file: %v and requirement.txt file: %v", condaYaml, requirementsText) - key, err := temporaryConfig(condaYaml, requirementsText, configurations...) - if err != nil { - failures += 1 - xviper.Set("stats.env.failures", failures) - return "", err +func pipLayer(fingerprint, requirementsText, targetFolder string, stopwatch fmt.Stringer, planWriter io.Writer) (bool, bool, bool, string) { + assertStageFolder(targetFolder) + common.TimelineBegin("Layer: pip [%s]", fingerprint) + defer common.TimelineEnd() + + pipUsed := false + fmt.Fprintf(planWriter, "\n--- pip plan @%ss ---\n\n", stopwatch) + python, pyok := FindPython(targetFolder) + if !pyok { + fmt.Fprintf(planWriter, "Note: no python in target folder: %s\n", targetFolder) + } + pipCache, wheelCache := common.PipCache(), common.WheelCache() + size, ok := pathlib.Size(requirementsText) + if !ok || size == 0 { + pretty.Progress(8, "Skipping pip install phase -- no pip dependencies.") + } else { + if !pyok { + cloud.InternalBackgroundMetric(common.ControllerIdentity(), "rcc.env.fatal.pip", fmt.Sprintf("%d_%x", 9999, 9999)) + common.Timeline("pip fail. no python found.") + common.Fatal("pip fail. no python found.", errors.New("No python found, but required!")) + return false, false, pipUsed, "" + } + pretty.Progress(8, "Running pip install phase. (pip v%s) [layer: %s]", PipVersion(python), fingerprint) + common.Debug("Updating new environment at %v with pip requirements from %v (size: %v)", targetFolder, requirementsText, size) + pipCommand := common.NewCommander(python, "-m", "pip", "install", "--isolated", "--no-color", "--disable-pip-version-check", "--prefer-binary", "--cache-dir", pipCache, "--find-links", wheelCache, "--requirement", requirementsText) + pipCommand.Option("--index-url", settings.Global.PypiURL()) + pipCommand.Option("--trusted-host", settings.Global.PypiTrustedHost()) + pipCommand.ConditionalFlag(common.VerboseEnvironmentBuilding(), "--verbose") + common.Debug("=== pip install phase ===") + code, err := LiveExecution(planWriter, targetFolder, pipCommand.CLI()...) + if err != nil || code != 0 { + cloud.InternalBackgroundMetric(common.ControllerIdentity(), "rcc.env.fatal.pip", fmt.Sprintf("%d_%x", code, code)) + common.Timeline("pip fail.") + common.Fatal(fmt.Sprintf("Pip [%d/%x]", code, code), err) + pretty.RccPointOfView(pipInstall, err) + return false, false, pipUsed, "" + } + journal.CurrentBuildEvent().PipComplete() + common.Timeline("pip done.") + pipUsed = true } - defer os.Remove(condaYaml) - defer os.Remove(requirementsText) - - liveFolder := LiveFrom(key) - if reuseExistingLive(key) { - hits += 1 - xviper.Set("stats.env.hit", hits) - return liveFolder, nil - } - common.Log("#### Progress: 1/4 [try clone existing same template to live, key: %v]", key) - if CloneFromTo(TemplateFrom(key), liveFolder) { - dirty += 1 - xviper.Set("stats.env.dirty", dirty) - return liveFolder, nil - } - common.Log("#### Progress: 2/4 [try create new environment from scratch]") - if newLive(condaYaml, requirementsText, key, force, freshInstall) { - misses += 1 - xviper.Set("stats.env.miss", misses) - common.Log("#### Progress: 3/4 [backup new environment as template]") - CloneFromTo(liveFolder, TemplateFrom(key)) - return liveFolder, nil - } - - failures += 1 - xviper.Set("stats.env.failures", failures) - return "", errors.New("Could not create environment.") + return true, false, pipUsed, python } -func RemoveEnvironment(label string) { - removeClone(LiveFrom(label)) - removeClone(TemplateFrom(label)) +func postInstallLayer(fingerprint string, postInstall []string, targetFolder string, stopwatch fmt.Stringer, planWriter io.Writer) (bool, bool) { + assertStageFolder(targetFolder) + common.TimelineBegin("Layer: post install scripts [%s]", fingerprint) + defer common.TimelineEnd() + + fmt.Fprintf(planWriter, "\n--- post install plan @%ss ---\n\n", stopwatch) + if postInstall != nil && len(postInstall) > 0 { + pretty.Progress(9, "Post install scripts phase started. [layer: %s]", fingerprint) + common.Debug("=== post install phase ===") + for _, script := range postInstall { + scriptCommand, err := shell.Split(script) + if err != nil { + common.Fatal("post-install", err) + common.Log("%sScript '%s' parsing failure: %v%s", pretty.Red, script, err, pretty.Reset) + pretty.RccPointOfView(postInstallScripts, err) + return false, false + } + common.Debug("Running post install script '%s' ...", script) + _, err = LiveExecution(planWriter, targetFolder, scriptCommand...) + if err != nil { + common.Fatal("post-install", err) + common.Log("%sScript '%s' failure: %v%s", pretty.Red, script, err, pretty.Reset) + pretty.RccPointOfView(postInstallScripts, err) + return false, false + } + } + journal.CurrentBuildEvent().PostInstallComplete() + } else { + pretty.Progress(9, "Post install scripts phase skipped -- no scripts.") + } + return true, false } -func removeClone(location string) { - os.Remove(metafile(location)) - os.RemoveAll(location) +func holotreeLayers(condaYaml, requirementsText string, finalEnv *Environment, targetFolder string, stopwatch fmt.Stringer, planWriter io.Writer, theplan *PlanWriter, force bool, skip SkipLayer, recorder Recorder) (bool, bool, bool, string) { + assertStageFolder(targetFolder) + common.TimelineBegin("Holotree layers at %q", targetFolder) + defer common.TimelineEnd() + + pipNeeded := len(requirementsText) > 0 + postInstall := len(finalEnv.PostInstall) > 0 + + var pypiSelector pipTool = pipLayer + + hasUv := finalEnv.HasCondaDependency("uv") + if hasUv { + pypiSelector = uvLayer + } + + layers := finalEnv.AsLayers() + fingerprints := finalEnv.FingerprintLayers() + + var success, fatal, pipUsed bool + var python string + + if skip < SkipMicromambaLayer { + success, fatal = micromambaLayer(fingerprints[0], condaYaml, targetFolder, stopwatch, planWriter, force) + if !success { + return success, fatal, false, "" + } + if pipNeeded || postInstall { + fmt.Fprintf(theplan, "\n--- micromamba layer complete [on layered holotree] ---\n\n") + common.Error("saving rcc_plan.log", theplan.Save()) + common.Error("saving golden master", goldenMaster(targetFolder, false)) + recorder.Record([]byte(layers[0])) + } + } else { + pretty.Progress(7, "Skipping micromamba phase, layer exists.") + fmt.Fprintf(planWriter, "\n--- micromamba plan skipped, layer exists ---\n\n") + } + if skip < SkipPipLayer { + success, fatal, pipUsed, python = pypiSelector(fingerprints[1], requirementsText, targetFolder, stopwatch, planWriter) + if !success { + return success, fatal, pipUsed, python + } + if pipUsed && postInstall { + fmt.Fprintf(theplan, "\n--- pip layer complete [on layered holotree] ---\n\n") + common.Error("saving rcc_plan.log", theplan.Save()) + common.Error("saving golden master", goldenMaster(targetFolder, true)) + recorder.Record([]byte(layers[1])) + } + } else { + pretty.Progress(8, "Skipping pip phase, layer exists.") + fmt.Fprintf(planWriter, "\n--- pip plan skiped, layer exists ---\n\n") + } + if skip < SkipPostinstallLayer { + success, fatal = postInstallLayer(fingerprints[2], finalEnv.PostInstall, targetFolder, stopwatch, planWriter) + if !success { + return success, fatal, pipUsed, python + } + } else { + pretty.Progress(9, "Skipping post install scripts phase, layer exists.") + fmt.Fprintf(planWriter, "\n--- post install plan skipped, layer exists ---\n\n") + } + return true, false, pipUsed, python } -func CloneFromTo(source, target string) bool { - removeClone(target) - os.MkdirAll(target, 0755) +func newLiveInternal(yaml, condaYaml, requirementsText, key string, force, freshInstall bool, skip SkipLayer, finalEnv *Environment, recorder Recorder) (bool, bool) { + targetFolder := common.StageFolder + theplan := NewPlanWriter(filepath.Join(targetFolder, "rcc_plan.log")) + failure := true + defer func() { + if failure { + common.Log("%s", theplan.AsText()) + } + }() + + planalyzer := NewPlanAnalyzer(true) + defer planalyzer.Close() + + planWriter := io.MultiWriter(theplan, planalyzer) + fmt.Fprintf(planWriter, "--- installation plan %q %s [force: %v, fresh: %v| rcc %s] ---\n\n", key, time.Now().Format(time.RFC3339), force, freshInstall, common.Version) + stopwatch := common.Stopwatch("installation plan") + fmt.Fprintf(planWriter, "--- plan blueprint @%ss ---\n\n", stopwatch) + fmt.Fprintf(planWriter, "%s\n", yaml) - if !IsPristine(source) { - removeClone(source) - return false + success, fatal, pipUsed, python := holotreeLayers(condaYaml, requirementsText, finalEnv, targetFolder, stopwatch, planWriter, theplan, force, skip, recorder) + if !success { + return success, fatal } - expected, err := metaLoad(source) + + pretty.Progress(10, "Activate environment started phase.") + common.Debug("=== activate phase ===") + fmt.Fprintf(planWriter, "\n--- activation plan @%ss ---\n\n", stopwatch) + err := Activate(planWriter, targetFolder) if err != nil { - return false + common.Log("%sActivation failure: %v%s", pretty.Yellow, err, pretty.Reset) } - success := cloneFolder(source, target, 8) - if !success { - removeClone(target) - return false + for _, line := range LoadActivationEnvironment(targetFolder) { + fmt.Fprintf(planWriter, "%s\n", line) } - digest, err := DigestFor(target) - if err != nil || Hexdigest(digest) != expected { - removeClone(target) - return false + err = goldenMaster(targetFolder, pipUsed) + if err != nil { + common.Log("%sGolden EE failure: %v%s", pretty.Yellow, err, pretty.Reset) + } + fmt.Fprintf(planWriter, "\n--- pip check plan @%ss ---\n\n", stopwatch) + if common.StrictFlag && pipUsed { + pretty.Progress(11, "Running pip check phase.") + pipCommand := common.NewCommander(python, "-m", "pip", "check", "--no-color") + pipCommand.ConditionalFlag(common.VerboseEnvironmentBuilding(), "--verbose") + common.Debug("=== pip check phase ===") + code, err := LiveExecution(planWriter, targetFolder, pipCommand.CLI()...) + if err != nil || code != 0 { + cloud.InternalBackgroundMetric(common.ControllerIdentity(), "rcc.env.fatal.pipcheck", fmt.Sprintf("%d_%x", code, code)) + common.Timeline("pip check fail.") + common.Fatal(fmt.Sprintf("Pip check [%d/%x]", code, code), err) + return false, false + } + common.Timeline("pip check done.") + } else { + pretty.Progress(11, "Pip check skipped.") } - metaSave(target, expected) - touchMetafile(source) - return true -} + fmt.Fprintf(planWriter, "\n--- installation plan complete @%ss ---\n\n", stopwatch) + pretty.Progress(12, "Update installation plan.") + common.Error("saving rcc_plan.log", theplan.Save()) + common.Debug("=== finalize phase ===") -func cloneFolder(source, target string, workers int) bool { - queue := make(chan copyRequest) - done := make(chan bool) + failure = false - for x := 0; x < workers; x++ { - go copyWorker(queue, done) - } - - success := copyFolder(source, target, queue) - close(queue) + return true, false +} - for x := 0; x < workers; x++ { - <-done +func LogUnifiedEnvironment(content []byte) { + environment, err := CondaYamlFrom(content) + if err != nil { + return } - - return success + yaml, err := environment.AsYaml() + if err != nil { + return + } + common.Log("FINAL unified conda environment descriptor:\n---\n%v---", yaml) } -func SilentTouch(directory string, when time.Time) bool { - handle, err := os.Open(directory) +func finalUnifiedEnvironment(filename string) (string, *Environment, error) { + right, err := ReadPackageCondaYaml(filename) if err != nil { - return false + return "", nil, err } - entries, err := handle.Readdir(-1) - handle.Close() + yaml, err := right.AsYaml() if err != nil { - return false + return "", nil, err } - for _, entry := range entries { - if !entry.IsDir() { - pathlib.TouchWhen(filepath.Join(directory, entry.Name()), when) - } + return yaml, right, nil +} + +func temporaryConfig(condaYaml, requirementsText, filename string) (string, string, *Environment, error) { + yaml, right, err := finalUnifiedEnvironment(filename) + if err != nil { + return "", "", nil, err } - return true + hash := common.ShortDigest(yaml) + err = right.SaveAsRequirements(requirementsText) + if err != nil { + return "", "", nil, err + } + pure := right.AsPureConda() + err = pure.SaveAs(condaYaml) + return hash, yaml, right, err } -func copyFolder(source, target string, queue chan copyRequest) bool { - os.Mkdir(target, 0755) +func LegacyEnvironment(recorder Recorder, force bool, skip SkipLayer, configuration string) error { + cloud.InternalBackgroundMetric(common.ControllerIdentity(), "rcc.env.create.start", common.Version) - handle, err := os.Open(source) + lockfile := common.ProductLock() + completed := pathlib.LockWaitMessage(lockfile, "Serialized environment creation [robocorp lock]") + locker, err := pathlib.Locker(lockfile, 30000, false) + completed() if err != nil { - common.Error("OPEN", err) - return false + common.Log("Could not get lock on live environment. Quitting!") + return err } - entries, err := handle.Readdir(-1) - handle.Close() + defer locker.Release() + + freshInstall := true + + condaYaml := filepath.Join(pathlib.TempDir(), fmt.Sprintf("conda_%x.yaml", common.When)) + requirementsText := filepath.Join(pathlib.TempDir(), fmt.Sprintf("require_%x.txt", common.When)) + common.Debug("Using temporary conda.yaml file: %v and requirement.txt file: %v", condaYaml, requirementsText) + key, yaml, finalEnv, err := temporaryConfig(condaYaml, requirementsText, configuration) if err != nil { - common.Error("DIR", err) - return false + return err } + defer os.Remove(condaYaml) + defer os.Remove(requirementsText) - success := true - expect := 0 - for _, entry := range entries { - if entry.Name() == "__pycache__" { - continue - } - newSource := filepath.Join(source, entry.Name()) - newTarget := filepath.Join(target, entry.Name()) - if entry.IsDir() { - copyFolder(newSource, newTarget, queue) - } else { - queue <- copyRequest{newSource, newTarget} - expect += 1 - } + success, err := newLive(yaml, condaYaml, requirementsText, key, force, freshInstall, skip, finalEnv, recorder) + if err != nil { + return err + } + if success { + return nil } - return success -} - -type copyRequest struct { - source, target string + return errors.New("Could not create environment.") } -func copyWorker(tasks chan copyRequest, done chan bool) { - for { - task, ok := <-tasks - if !ok { - break - } - link, err := os.Readlink(task.source) - if err != nil { - pathlib.CopyFile(task.source, task.target, false) - continue - } - err = os.Symlink(link, task.target) - if err != nil { - common.Error("LINK", err) - continue - } +func renameRemove(location string) error { + if !pathlib.IsDir(location) { + common.Trace("Location %q is not directory, not removed.", location) + return nil } - - done <- true + randomLocation := fmt.Sprintf("%s.%08X", location, rand.Uint32()) + common.Debug("Rename/remove %q using %q as random name.", location, randomLocation) + err := os.Rename(location, randomLocation) + if err != nil { + common.Log("Rename %q -> %q failed as: %v!", location, randomLocation, err) + return err + } + common.Trace("Rename %q -> %q was successful!", location, randomLocation) + err = os.RemoveAll(randomLocation) + if err != nil { + common.Log("Removal of %q failed as: %v!", randomLocation, err) + return err + } + common.Trace("Removal of %q was successful!", randomLocation) + meta := metafile(location) + if pathlib.IsFile(meta) { + err = os.Remove(meta) + common.Trace("Removal of %q result was %v.", meta, err) + return err + } + common.Trace("Metafile %q was not file.", meta) + return nil } diff --git a/developer/.gitignore b/developer/.gitignore new file mode 100644 index 00000000..811ab3fe --- /dev/null +++ b/developer/.gitignore @@ -0,0 +1,17 @@ +output/ +venv/ +temp/ +.rpa/ +.idea/ +.ipynb_checkpoints/ +*/.ipynb_checkpoints/* +.virtual_documents/ +*/.virtual_documents/* +.vscode +.DS_Store +*.pyc +*.zip +*/work-items-out/* +testrun/* +.~lock* +*.pkl diff --git a/developer/README.md b/developer/README.md new file mode 100644 index 00000000..ca8a4cd1 --- /dev/null +++ b/developer/README.md @@ -0,0 +1,55 @@ +# Developer setup helper + +To give idea, what is needed to develop rcc. This is bootstrapping rcc +development with older version of rcc. So, you really need older rcc +installed somewhere available in PATH. + +This developer toolkit uses both `tasks:` and `devTasks:` to enable tools. +Pay attention for `--dev` flag usage. + +## One task to test the thing with robot + +``` +rcc run -r developer/toolkit.yaml -t robot +``` + +Then see `tmp/output/log.html` for possible failure details. + +## Some developer tasks + +### Unit tests + +``` +rcc run -r developer/toolkit.yaml --dev -t unitTests +``` + +You can also run tests running `invoke` directly from your CLI, or run `go test` - when running unit tests +outside of `invoke` however, make sure `GOARCH` env variable is set to `amd64`, as some tests may rely on it. + +### Building the thing for local OS + +``` +rcc run -r developer/toolkit.yaml --dev -t local +``` + +### Building the thing (all OSes) + +``` +rcc run -r developer/toolkit.yaml --dev -t build +``` + +### Update documentation TOC + +``` +rcc run -r developer/toolkit.yaml --dev -t toc +``` + +### Show tools + +``` +rcc run -r developer/toolkit.yaml --dev -t tools +``` + +## Dependencies + +Needed dependencies are visible at `developer/setup.yaml` file. diff --git a/developer/call_invoke.py b/developer/call_invoke.py new file mode 100644 index 00000000..c604ba52 --- /dev/null +++ b/developer/call_invoke.py @@ -0,0 +1,9 @@ +import os +import subprocess +import sys + +use_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +assert len(sys.argv) >= 2, "No task provided when calling `call_invoke.py`" +task = sys.argv[1] +exit(subprocess.run(("invoke", task), cwd=use_dir).returncode) diff --git a/developer/setup.yaml b/developer/setup.yaml new file mode 100644 index 00000000..b955a26b --- /dev/null +++ b/developer/setup.yaml @@ -0,0 +1,12 @@ +channels: + - conda-forge +dependencies: + # Note: needs to match the version in the GitHub Actions workflow + # (both rcc and rcc-pipeline) + - python=3.10.15 + - invoke=2.2.0 + # Note: needs to match the version in robot_requirements.txt + # Also in rcc-pipeline + - robotframework=6.1.1 + - go=1.20.7 + - git=2.46.0 diff --git a/developer/toolkit.yaml b/developer/toolkit.yaml new file mode 100644 index 00000000..bd9fcc1b --- /dev/null +++ b/developer/toolkit.yaml @@ -0,0 +1,25 @@ +tasks: + robot: + shell: python call_invoke.py robot + +devTasks: + unitTests: + shell: python call_invoke.py test + build: + shell: python call_invoke.py build + local: + shell: python call_invoke.py local + tools: + shell: python call_invoke.py tooling + toc: + shell: python call_invoke.py toc + +environmentConfigs: + - setup.yaml + +artifactsDir: tmp + +PATH: +PYTHONPATH: +ignoreFiles: + - .gitignore diff --git a/docs/BUILD.md b/docs/BUILD.md index 9960660a..08e80147 100644 --- a/docs/BUILD.md +++ b/docs/BUILD.md @@ -5,9 +5,10 @@ Required tools are: - golang for implementing the thing -- rake for automating building the thing +- invoke for automating building the thing - robot for testing the thing -- zip to build template zipfiles + +See also: developer/README.md and developer/setup.yaml Internal requirements: @@ -15,14 +16,14 @@ Internal requirements: ## Commands -- to see available tasks, use `rake -T` -- to build everything, use `rake build` command -- to run robot tests, use `rake robot` command -- note, that most of rake commands are build to be used in Github Actions +- to see available tasks, use `inv -l` +- to build everything, use `inv build` command +- to run robot tests, use `inv robot` command +- note, that most of invoke commands are built to be used in Github Actions ## Where to start reading code? To get started with CLI, start from "cmd" directory, which contains commands executed from CLI, each in separate file (plus additional support files). From there, use your editors code navigation to get to actual underlying -functions. \ No newline at end of file +functions. diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 00000000..b871113a --- /dev/null +++ b/docs/README.md @@ -0,0 +1,150 @@ +# Table of contents: rcc documentation +## 1 [Incomplete list of rcc use cases](https://github.com/robocorp/rcc/blob/master/docs/usecases.md#incomplete-list-of-rcc-use-cases) +### 1.1 [What is available from conda-forge?](https://github.com/robocorp/rcc/blob/master/docs/usecases.md#what-is-available-from-conda-forge) +## 2 [Incomplete list of rcc features](https://github.com/robocorp/rcc/blob/master/docs/features.md#incomplete-list-of-rcc-features) +## 3 [Tips, tricks, and recipies](https://github.com/robocorp/rcc/blob/master/docs/recipes.md#tips-tricks-and-recipies) +### 3.1 [How to see dependency changes?](https://github.com/robocorp/rcc/blob/master/docs/recipes.md#how-to-see-dependency-changes) +#### 3.1.1 [Why is this important?](https://github.com/robocorp/rcc/blob/master/docs/recipes.md#why-is-this-important) +#### 3.1.2 [Example of dependencies listing from holotree environment](https://github.com/robocorp/rcc/blob/master/docs/recipes.md#example-of-dependencies-listing-from-holotree-environment) +### 3.2 [How to freeze dependencies?](https://github.com/robocorp/rcc/blob/master/docs/recipes.md#how-to-freeze-dependencies) +#### 3.2.1 [Steps](https://github.com/robocorp/rcc/blob/master/docs/recipes.md#steps) +#### 3.2.2 [Limitations](https://github.com/robocorp/rcc/blob/master/docs/recipes.md#limitations) +### 3.3 [How pass arguments to robot from CLI?](https://github.com/robocorp/rcc/blob/master/docs/recipes.md#how-pass-arguments-to-robot-from-cli) +#### 3.3.1 [Example robot.yaml with scripting task](https://github.com/robocorp/rcc/blob/master/docs/recipes.md#example-robotyaml-with-scripting-task) +#### 3.3.2 [Run it with `--` separator.](https://github.com/robocorp/rcc/blob/master/docs/recipes.md#run-it-with----separator) +### 3.4 [How to run any command inside robot environment?](https://github.com/robocorp/rcc/blob/master/docs/recipes.md#how-to-run-any-command-inside-robot-environment) +#### 3.4.1 [Some example commands](https://github.com/robocorp/rcc/blob/master/docs/recipes.md#some-example-commands) +### 3.5 [How to convert existing python project to rcc?](https://github.com/robocorp/rcc/blob/master/docs/recipes.md#how-to-convert-existing-python-project-to-rcc) +#### 3.5.1 [Basic workflow to get it up and running](https://github.com/robocorp/rcc/blob/master/docs/recipes.md#basic-workflow-to-get-it-up-and-running) +#### 3.5.2 [What next?](https://github.com/robocorp/rcc/blob/master/docs/recipes.md#what-next) +### 3.6 [Is rcc limited to Python and Robot Framework?](https://github.com/robocorp/rcc/blob/master/docs/recipes.md#is-rcc-limited-to-python-and-robot-framework) +#### 3.6.1 [This is what we are going to do ...](https://github.com/robocorp/rcc/blob/master/docs/recipes.md#this-is-what-we-are-going-to-do-) +#### 3.6.2 [Write a robot.yaml](https://github.com/robocorp/rcc/blob/master/docs/recipes.md#write-a-robotyaml) +#### 3.6.3 [Write a conda.yaml](https://github.com/robocorp/rcc/blob/master/docs/recipes.md#write-a-condayaml) +#### 3.6.4 [Write a bin/builder.sh](https://github.com/robocorp/rcc/blob/master/docs/recipes.md#write-a-binbuildersh) +### 3.7 [Think what you can do with this conda.yaml?](https://github.com/robocorp/rcc/blob/master/docs/recipes.md#think-what-you-can-do-with-this-condayaml) +### 3.8 [How to control holotree environments?](https://github.com/robocorp/rcc/blob/master/docs/recipes.md#how-to-control-holotree-environments) +#### 3.8.1 [How to get understanding on holotree?](https://github.com/robocorp/rcc/blob/master/docs/recipes.md#how-to-get-understanding-on-holotree) +#### 3.8.2 [How to activate holotree environment?](https://github.com/robocorp/rcc/blob/master/docs/recipes.md#how-to-activate-holotree-environment) +### 3.9 [What is `ROBOCORP_HOME`?](https://github.com/robocorp/rcc/blob/master/docs/recipes.md#what-is-robocorp_home) +#### 3.9.1 [Are there some rules for `ROBOCORP_HOME` variable?](https://github.com/robocorp/rcc/blob/master/docs/recipes.md#are-there-some-rules-for-robocorp_home-variable) +#### 3.9.2 [When you might actually need to setup `ROBOCORP_HOME`?](https://github.com/robocorp/rcc/blob/master/docs/recipes.md#when-you-might-actually-need-to-setup-robocorp_home) +### 3.10 [What is shared holotree?](https://github.com/robocorp/rcc/blob/master/docs/recipes.md#what-is-shared-holotree) +### 3.11 [How to setup rcc to use shared holotree?](https://github.com/robocorp/rcc/blob/master/docs/recipes.md#how-to-setup-rcc-to-use-shared-holotree) +#### 3.11.1 [One time setup](https://github.com/robocorp/rcc/blob/master/docs/recipes.md#one-time-setup) +#### 3.11.2 [Reverting back to private holotrees](https://github.com/robocorp/rcc/blob/master/docs/recipes.md#reverting-back-to-private-holotrees) +### 3.12 [What can be controlled using environment variables?](https://github.com/robocorp/rcc/blob/master/docs/recipes.md#what-can-be-controlled-using-environment-variables) +### 3.13 [How to troubleshoot rcc setup and robots?](https://github.com/robocorp/rcc/blob/master/docs/recipes.md#how-to-troubleshoot-rcc-setup-and-robots) +#### 3.13.1 [Additional debugging options](https://github.com/robocorp/rcc/blob/master/docs/recipes.md#additional-debugging-options) +### 3.14 [Advanced network diagnostics](https://github.com/robocorp/rcc/blob/master/docs/recipes.md#advanced-network-diagnostics) +#### 3.14.1 [Configuration](https://github.com/robocorp/rcc/blob/master/docs/recipes.md#configuration) +### 3.15 [What is in `robot.yaml`?](https://github.com/robocorp/rcc/blob/master/docs/recipes.md#what-is-in-robotyaml) +#### 3.15.1 [Example](https://github.com/robocorp/rcc/blob/master/docs/recipes.md#example) +#### 3.15.2 [What is this `robot.yaml` thing?](https://github.com/robocorp/rcc/blob/master/docs/recipes.md#what-is-this-robotyaml-thing) +#### 3.15.3 [Why "the center of the universe"?](https://github.com/robocorp/rcc/blob/master/docs/recipes.md#why-the-center-of-the-universe) +#### 3.15.4 [What are `tasks:`?](https://github.com/robocorp/rcc/blob/master/docs/recipes.md#what-are-tasks) +#### 3.15.5 [What are `devTasks:`?](https://github.com/robocorp/rcc/blob/master/docs/recipes.md#what-are-devtasks) +#### 3.15.6 [What is `condaConfigFile:`?](https://github.com/robocorp/rcc/blob/master/docs/recipes.md#what-is-condaconfigfile) +#### 3.15.7 [What are `environmentConfigs:`?](https://github.com/robocorp/rcc/blob/master/docs/recipes.md#what-are-environmentconfigs) +#### 3.15.8 [What are `preRunScripts:`?](https://github.com/robocorp/rcc/blob/master/docs/recipes.md#what-are-prerunscripts) +#### 3.15.9 [What is `artifactsDir:`?](https://github.com/robocorp/rcc/blob/master/docs/recipes.md#what-is-artifactsdir) +#### 3.15.10 [What are `ignoreFiles:`?](https://github.com/robocorp/rcc/blob/master/docs/recipes.md#what-are-ignorefiles) +#### 3.15.11 [What are `PATH:`?](https://github.com/robocorp/rcc/blob/master/docs/recipes.md#what-are-path) +#### 3.15.12 [What are `PYTHONPATH:`?](https://github.com/robocorp/rcc/blob/master/docs/recipes.md#what-are-pythonpath) +### 3.16 [What is in `conda.yaml`?](https://github.com/robocorp/rcc/blob/master/docs/recipes.md#what-is-in-condayaml) +#### 3.16.1 [Example](https://github.com/robocorp/rcc/blob/master/docs/recipes.md#example) +#### 3.16.2 [What is this `conda.yaml` thing?](https://github.com/robocorp/rcc/blob/master/docs/recipes.md#what-is-this-condayaml-thing) +#### 3.16.3 [What are `channels:`?](https://github.com/robocorp/rcc/blob/master/docs/recipes.md#what-are-channels) +#### 3.16.4 [What are `dependencies:`?](https://github.com/robocorp/rcc/blob/master/docs/recipes.md#what-are-dependencies) +#### 3.16.5 [What are `rccPostInstall:` scripts?](https://github.com/robocorp/rcc/blob/master/docs/recipes.md#what-are-rccpostinstall-scripts) +### 3.17 [How to do "old-school" CI/CD pipeline integration with rcc?](https://github.com/robocorp/rcc/blob/master/docs/recipes.md#how-to-do-old-school-cicd-pipeline-integration-with-rcc) +#### 3.17.1 [The oldschoolci.sh script](https://github.com/robocorp/rcc/blob/master/docs/recipes.md#the-oldschoolcish-script) +#### 3.17.2 [A setup.sh script for simulating variable injection.](https://github.com/robocorp/rcc/blob/master/docs/recipes.md#a-setupsh-script-for-simulating-variable-injection) +#### 3.17.3 [Simulating actual CI/CD step in local machine.](https://github.com/robocorp/rcc/blob/master/docs/recipes.md#simulating-actual-cicd-step-in-local-machine) +#### 3.17.4 [Additional notes](https://github.com/robocorp/rcc/blob/master/docs/recipes.md#additional-notes) +### 3.18 [How to setup custom templates?](https://github.com/robocorp/rcc/blob/master/docs/recipes.md#how-to-setup-custom-templates) +#### 3.18.1 [Custom template configuration in `settings.yaml`.](https://github.com/robocorp/rcc/blob/master/docs/recipes.md#custom-template-configuration-in-settingsyaml-) +#### 3.18.2 [Custom template configuration file as `templates.yaml`.](https://github.com/robocorp/rcc/blob/master/docs/recipes.md#custom-template-configuration-file-as-templatesyaml-) +#### 3.18.3 [Custom template content in `templates.zip` file.](https://github.com/robocorp/rcc/blob/master/docs/recipes.md#custom-template-content-in-templateszip-file) +#### 3.18.4 [Shared using `https:` protocol ...](https://github.com/robocorp/rcc/blob/master/docs/recipes.md#shared-using-https-protocol-) +### 3.19 [Where can I find updates for rcc?](https://github.com/robocorp/rcc/blob/master/docs/recipes.md#where-can-i-find-updates-for-rcc) +### 3.20 [What has changed on rcc?](https://github.com/robocorp/rcc/blob/master/docs/recipes.md#what-has-changed-on-rcc) +#### 3.20.1 [See changelog from git repo ...](https://github.com/robocorp/rcc/blob/master/docs/recipes.md#see-changelog-from-git-repo-) +#### 3.20.2 [See that from your version of rcc directly ...](https://github.com/robocorp/rcc/blob/master/docs/recipes.md#see-that-from-your-version-of-rcc-directly-) +### 3.21 [Can I see these tips as web page?](https://github.com/robocorp/rcc/blob/master/docs/recipes.md#can-i-see-these-tips-as-web-page) +## 4 [Profile Configuration](https://github.com/robocorp/rcc/blob/master/docs/profile_configuration.md#profile-configuration) +### 4.1 [What is profile?](https://github.com/robocorp/rcc/blob/master/docs/profile_configuration.md#what-is-profile) +#### 4.1.1 [When do you need profiles?](https://github.com/robocorp/rcc/blob/master/docs/profile_configuration.md#when-do-you-need-profiles) +#### 4.1.2 [What does it contain?](https://github.com/robocorp/rcc/blob/master/docs/profile_configuration.md#what-does-it-contain) +### 4.2 [Quick start guide](https://github.com/robocorp/rcc/blob/master/docs/profile_configuration.md#quick-start-guide) +#### 4.2.1 [Setup Utility -- user interface for this](https://github.com/robocorp/rcc/blob/master/docs/profile_configuration.md#setup-utility----user-interface-for-this) +#### 4.2.2 [Pure rcc workflow for handling existing profiles](https://github.com/robocorp/rcc/blob/master/docs/profile_configuration.md#pure-rcc-workflow-for-handling-existing-profiles) +### 4.3 [What is needed?](https://github.com/robocorp/rcc/blob/master/docs/profile_configuration.md#what-is-needed) +### 4.4 [Discovery process](https://github.com/robocorp/rcc/blob/master/docs/profile_configuration.md#discovery-process) +### 4.5 [What is execution environment isolation and caching?](https://github.com/robocorp/rcc/blob/master/docs/environment-caching.md#what-is-execution-environment-isolation-and-caching) +### 4.6 [The second evolution of environment management in RCC](https://github.com/robocorp/rcc/blob/master/docs/environment-caching.md#the-second-evolution-of-environment-management-in-rcc) +#### 4.6.1 [Relocation and file locking](https://github.com/robocorp/rcc/blob/master/docs/environment-caching.md#relocation-and-file-locking) +### 4.7 [A better analogy: accommodations](https://github.com/robocorp/rcc/blob/master/docs/environment-caching.md#a-better-analogy-accommodations) +#### 4.7.1 ["I invited you to my home."](https://github.com/robocorp/rcc/blob/master/docs/environment-caching.md#i-invited-you-to-my-home) +#### 4.7.2 ["Welcome to a hotel built out of ship containers."](https://github.com/robocorp/rcc/blob/master/docs/environment-caching.md#welcome-to-a-hotel-built-out-of-ship-containers) +#### 4.7.3 ["Welcome to an actual Hotel."](https://github.com/robocorp/rcc/blob/master/docs/environment-caching.md#welcome-to-an-actual-hotel) +## 5 [Holotree and library maintenance](https://github.com/robocorp/rcc/blob/master/docs/maintenance.md#holotree-and-library-maintenance) +### 5.1 [Why do maintenance?](https://github.com/robocorp/rcc/blob/master/docs/maintenance.md#why-do-maintenance) +### 5.2 [Shared holotree and maintenance](https://github.com/robocorp/rcc/blob/master/docs/maintenance.md#shared-holotree-and-maintenance) +### 5.3 [Maintenance vs. tools using holotrees](https://github.com/robocorp/rcc/blob/master/docs/maintenance.md#maintenance-vs-tools-using-holotrees) +### 5.4 [Maintenace and product families](https://github.com/robocorp/rcc/blob/master/docs/maintenance.md#maintenace-and-product-families) +### 5.5 [Deleting catalogs and spaces](https://github.com/robocorp/rcc/blob/master/docs/maintenance.md#deleting-catalogs-and-spaces) +### 5.6 [Keeping hololib consistent](https://github.com/robocorp/rcc/blob/master/docs/maintenance.md#keeping-hololib-consistent) +### 5.7 [Summary of maintenance related commands](https://github.com/robocorp/rcc/blob/master/docs/maintenance.md#summary-of-maintenance-related-commands) +## 6 [Support for virtual environments](https://github.com/robocorp/rcc/blob/master/docs/venv.md#support-for-virtual-environments) +### 6.1 [What does it do?](https://github.com/robocorp/rcc/blob/master/docs/venv.md#what-does-it-do) +### 6.2 [How to get started?](https://github.com/robocorp/rcc/blob/master/docs/venv.md#how-to-get-started) +### 6.3 [Limitations of `rcc venv`:](https://github.com/robocorp/rcc/blob/master/docs/venv.md#limitations-of-rcc-venv) +### 6.4 [Dangers of using `--force` in `rcc venv` context.](https://github.com/robocorp/rcc/blob/master/docs/venv.md#dangers-of-using---force-in-rcc-venv-context) +### 6.5 [What is this `depxtraction.py` thing?](https://github.com/robocorp/rcc/blob/master/docs/venv.md#what-is-this-depxtractionpy-thing) +### 6.6 [Limitations of `depxtraction.py`:](https://github.com/robocorp/rcc/blob/master/docs/venv.md#limitations-of-depxtractionpy) +### 6.7 [Ideas for usage](https://github.com/robocorp/rcc/blob/master/docs/venv.md#ideas-for-usage) +## 7 [Troubleshooting guidelines and known solutions](https://github.com/robocorp/rcc/blob/master/docs/troubleshooting.md#troubleshooting-guidelines-and-known-solutions) +### 7.1 [Tools to help with troubleshooting issues](https://github.com/robocorp/rcc/blob/master/docs/troubleshooting.md#tools-to-help-with-troubleshooting-issues) +### 7.2 [How to troubleshoot issue you are having?](https://github.com/robocorp/rcc/blob/master/docs/troubleshooting.md#how-to-troubleshoot-issue-you-are-having) +### 7.3 [Reporting an issue](https://github.com/robocorp/rcc/blob/master/docs/troubleshooting.md#reporting-an-issue) +### 7.4 [Network access related troubleshooting questions](https://github.com/robocorp/rcc/blob/master/docs/troubleshooting.md#network-access-related-troubleshooting-questions) +### 7.5 [Known solutions](https://github.com/robocorp/rcc/blob/master/docs/troubleshooting.md#known-solutions) +#### 7.5.1 [Access denied while building holotree environment (Windows)](https://github.com/robocorp/rcc/blob/master/docs/troubleshooting.md#access-denied-while-building-holotree-environment-windows) +#### 7.5.2 [Message "Serialized environment creation" repeats](https://github.com/robocorp/rcc/blob/master/docs/troubleshooting.md#message-serialized-environment-creation-repeats) +## 8 [Vocabulary](https://github.com/robocorp/rcc/blob/master/docs/vocabulary.md#vocabulary) +### 8.1 [Blueprint](https://github.com/robocorp/rcc/blob/master/docs/vocabulary.md#blueprint) +### 8.2 [Catalog](https://github.com/robocorp/rcc/blob/master/docs/vocabulary.md#catalog) +### 8.3 [Controller](https://github.com/robocorp/rcc/blob/master/docs/vocabulary.md#controller) +### 8.4 [Diagnostics](https://github.com/robocorp/rcc/blob/master/docs/vocabulary.md#diagnostics) +### 8.5 [Dirty environment](https://github.com/robocorp/rcc/blob/master/docs/vocabulary.md#dirty-environment) +### 8.6 [Environment](https://github.com/robocorp/rcc/blob/master/docs/vocabulary.md#environment) +### 8.7 [Fingerprint](https://github.com/robocorp/rcc/blob/master/docs/vocabulary.md#fingerprint) +### 8.8 [Holotree](https://github.com/robocorp/rcc/blob/master/docs/vocabulary.md#holotree) +### 8.9 [Hololib](https://github.com/robocorp/rcc/blob/master/docs/vocabulary.md#hololib) +### 8.10 [Identity](https://github.com/robocorp/rcc/blob/master/docs/vocabulary.md#identity) +### 8.11 [Platform](https://github.com/robocorp/rcc/blob/master/docs/vocabulary.md#platform) +### 8.12 [Prebuild environment](https://github.com/robocorp/rcc/blob/master/docs/vocabulary.md#prebuild-environment) +### 8.13 [Pristine environment](https://github.com/robocorp/rcc/blob/master/docs/vocabulary.md#pristine-environment) +### 8.14 [Private holotree](https://github.com/robocorp/rcc/blob/master/docs/vocabulary.md#private-holotree) +### 8.15 [Product family](https://github.com/robocorp/rcc/blob/master/docs/vocabulary.md#product-family) +### 8.16 [Profile](https://github.com/robocorp/rcc/blob/master/docs/vocabulary.md#profile) +### 8.17 [Robot](https://github.com/robocorp/rcc/blob/master/docs/vocabulary.md#robot) +### 8.18 [Shared holotree](https://github.com/robocorp/rcc/blob/master/docs/vocabulary.md#shared-holotree) +### 8.19 [Space](https://github.com/robocorp/rcc/blob/master/docs/vocabulary.md#space) +### 8.20 [Unmanaged holotree space](https://github.com/robocorp/rcc/blob/master/docs/vocabulary.md#unmanaged-holotree-space) +### 8.21 [User](https://github.com/robocorp/rcc/blob/master/docs/vocabulary.md#user) +## 9 [History of rcc](https://github.com/robocorp/rcc/blob/master/docs/history.md#history-of-rcc) +### 9.1 [Version 11.x: between Sep 6, 2021 and ...](https://github.com/robocorp/rcc/blob/master/docs/history.md#version-11x-between-sep-6-2021-and-) +### 9.2 [Version 10.x: between Jun 15, 2021 and Sep 1, 2021](https://github.com/robocorp/rcc/blob/master/docs/history.md#version-10x-between-jun-15-2021-and-sep-1-2021) +### 9.3 [Version 9.x: between Jan 15, 2021 and Jun 10, 2021](https://github.com/robocorp/rcc/blob/master/docs/history.md#version-9x-between-jan-15-2021-and-jun-10-2021) +### 9.4 [Version 8.x: between Jan 4, 2021 and Jan 18, 2021](https://github.com/robocorp/rcc/blob/master/docs/history.md#version-8x-between-jan-4-2021-and-jan-18-2021) +### 9.5 [Version 7.x: between Dec 1, 2020 and Jan 4, 2021](https://github.com/robocorp/rcc/blob/master/docs/history.md#version-7x-between-dec-1-2020-and-jan-4-2021) +### 9.6 [Version 6.x: between Nov 16, 2020 and Nov 30, 2020](https://github.com/robocorp/rcc/blob/master/docs/history.md#version-6x-between-nov-16-2020-and-nov-30-2020) +### 9.7 [Version 5.x: between Nov 4, 2020 and Nov 16, 2020](https://github.com/robocorp/rcc/blob/master/docs/history.md#version-5x-between-nov-4-2020-and-nov-16-2020) +### 9.8 [Version 4.x: between Oct 20, 2020 and Nov 2, 2020](https://github.com/robocorp/rcc/blob/master/docs/history.md#version-4x-between-oct-20-2020-and-nov-2-2020) +### 9.9 [Version 3.x: between Oct 15, 2020 and Oct 19, 2020](https://github.com/robocorp/rcc/blob/master/docs/history.md#version-3x-between-oct-15-2020-and-oct-19-2020) +### 9.10 [Version 2.x: between Sep 16, 2020 and Oct 14, 2020](https://github.com/robocorp/rcc/blob/master/docs/history.md#version-2x-between-sep-16-2020-and-oct-14-2020) +### 9.11 [Version 1.x: between Sep 3, 2020 and Sep 16, 2020](https://github.com/robocorp/rcc/blob/master/docs/history.md#version-1x-between-sep-3-2020-and-sep-16-2020) +### 9.12 [Version 0.x: between April 1, 2020 and Sep 8, 2020](https://github.com/robocorp/rcc/blob/master/docs/history.md#version-0x-between-april-1-2020-and-sep-8-2020) +### 9.13 [Birth of "Codename: Conman"](https://github.com/robocorp/rcc/blob/master/docs/history.md#birth-of-codename-conman) \ No newline at end of file diff --git a/docs/changelog.md b/docs/changelog.md new file mode 100644 index 00000000..9daf4480 --- /dev/null +++ b/docs/changelog.md @@ -0,0 +1,2614 @@ +# rcc change log + +## v18.1.7 (date: 17.10.2024) + +- adding support for windows development of rcc +- using python/invoke instead of ruby/rake for building rcc +- code formatting python code with ruff +- also using windows runner for tests in github actions + +## v18.1.6 (date: 21.8.2024) + +- unit tests suite now works properly on MacOS and Windows + +## v18.1.5 (date: 7.8.2024) + +- developer directory tooling update + +## v18.1.4 (date: 5.8.2024) + +- bugfix: when there is PS1 in holotree variables, it is now filtered out +- developer directory with toolkit.yaml (hidden robot.yaml) + +## v18.1.3 (date: 2.8.2024) + +- bugfix: tlsCheck was giving nil TLS information without error +- CONTRIBUTING.md -- things to note when developing rcc + +## v18.1.2 (date: 28.6.2024) + +- updated default settings.yaml for Sema4.ai products. +- templates location also back in Sema4.ai settings.yaml. +- documentation updates + +## v18.1.1 (date: 26.6.2024) WORK IN PROGRESS + +- bug fix: too many commands were only visible with `--robocorp` product + strategy, but they are needed also in `--sema4ai` strategy + +## v18.1.0 (date: 26.6.2024) WORK IN PROGRESS + +- new command `feedback batch` for applications to send many metrics at once +- disabling rcc internal metrics based on product strategy +- bug fix: journal appending as more atomic operation (just one write) + +## v18.0.5 (date: 14.6.2024) WORK IN PROGRESS + +- MAJOR breaking change: now command `rcc configuration settings` will require + `--defaults` flag to show defaults template. Without it, default functionality + now is to show effective/active settings in YAML format. + +## v18.0.4 (date: 12.6.2024) WORK IN PROGRESS + +- Additional `--robocorp` product flag added. To match `--sema4ai` flag. +- Now using `%ProgramData%` instead of hard coded `c:\ProgramData\` in code. +- Update on default `settings.yaml` for Sema4.ai products. + +## v18.0.3 (date: 7.6.2024) WORK IN PROGRESS + +- Windows bugfix: icacls now applied on shared holotree location from product + strategy (was hardcoded before) + +## v18.0.2 (date: 7.6.2024) WORK IN PROGRESS + +- default `settings.yaml` is now behind product strategy (each product can + have their own default settings) + +## v18.0.1 (date: 5.6.2024) WORK IN PROGRESS + +- added company name as strategy name (dynamic name handling for user messages) +- replaced static "Robocorp" references with strategy name +- renamed some Robocorp functions to more generic Product functions + +## v18.0.0 (date: 3.6.2024) WORK IN PROGRESS + +- MAJOR breaking change: rcc will now live in two product domains, + Robocorp and Sema4.ai +- feature: initial support for `--sema4ai` strategy selection +- robot tests to test Sema4.ai support + +## v17.29.1 (date: 29.5.2024) + +- bugfix: when taking locks, some of those need to be in shared directory, + while others should not; code was making too much directories shared + +## v17.29.0 (date: 27.5.2024) + +- bugfix: removing `VIRTUAL_ENV` when rcc is executing subprocesses +- adding warning about that environment variable also in diagnostics + +## v17.28.4 (date: 26.4.2024) + +- bugfix: when there is "rcc point of view" message, it was not showing + who was controller, so now controller is visible + +## v17.28.3 (date: 26.4.2024) + +- bugfix: metrics sending was stating things as error, but they are not + critical (so that is now mentioned in message) + +## v17.28.2 (date: 24.4.2024) + +- bugfix: more places are now using package/conda YAML loading + +## v17.28.1 (date: 24.4.2024) + +- bugfix: when exporting prebuild environments, include layer catalogs also +- bugfix: exporting was not adding all environments correctly to .zip file + +## v17.28.0 (date: 22.4.2024) + +- adding support for `package.yaml` as replacement for `conda.yaml` + +## v17.27.0 (date: 17.4.2024) + +- when pip dependencies has `--use-feature=truststore` those environments + are identified as cacheable +- removed some robot.yaml file diagnostic checks since those are not valid + anymore + +## v17.26.0 (date: 17.4.2024) + +- feature: `--no-retry-build` flag for tools to prevent rcc doing retry + environment build in case of first build fails + +## v17.25.0 (date: 17.4.2024) + +- bug: when first build failed, original layers were expected to still be there +- fix: now second build always builds all layers (since failure needs that) + +## v17.24.0 (date: 15.4.2024) + +- micromamba upgrade to v1.5.8 + +## v17.23.2 (date: 15.4.2024) + +- more github action upgrades + +## v17.23.1 (date: 15.4.2024) + +- github action upgrades + +## v17.23.0 (date: 10.4.2024) + +- cleanup improvement with option `--caches` to remove conda/pip/uv/hololib + caches but leave holotree available +- also environment building now cleans up "building" space both before and + after environment is build + +## v17.22.0 (date: 27.3.2024) WORK IN PROGRESS + +- compression flag is now globally accessible +- compression flag also switches using siphash as identity hasher +- dirtyness stats also now lists duplicates and linked files + +## v17.21.0 (date: 25.3.2024) WORK IN PROGRESS + +- experimental feature to disable compression of hololib content +- made cleanup much more strict on error detection +- bug fix: task shell was missing `--space` option; added now + +## v17.20.0 (date: 12.3.2024) WORK IN PROGRESS + +- uv experiment with limited scope and imperfect implementation + +## v17.19.0 (date: 11.3.2024) + +- micromamba upgrade to v1.5.7 + +## v17.18.1 (date: 4.3.2024) + +- bugfix: template hash case-sensitivity fix + +## v17.18.0 (date: 23.2.2024) + +- new `--bundled` flag to support cases where rcc is bundled inside other apps +- first thing behind "bundled" flag is version check (so when flag is given, + rcc will never check possible newer version existence) +- bugfix: flag handling defaults on peek initialized flags +- typofix: on certificate appending failure message + +## v17.17.4 (date: 19.2.2024) + +- added `venv.md` for start of documentation for `rcc venv` and depxtraction + tooling and ideas how to use them +- added new man page command into rcc: `rcc man venv` + +## v17.17.3 (date: 17.2.2024) + +- bugfix: unwanted logging output can now be hidden using global option + `--log-hide ` (and can be given multiple times) + +## v17.17.2 (date: 14.2.2024) + +- depxtraction output update and refactoring code + +## v17.17.1 (date: 12.2.2024) + +- fixed space .use file to be written only when path is actually known + +## v17.17.0 (date: 9.2.2024) + +- adding depxtraction as part of `rcc holotree venv` creation + +## v17.16.0 (date: 7.2.2024) + +- new `NO_PROXY` configuration addition to `settings.yaml` file. +- that `NO_PROXY` will override previous OS level configuration, so be careful +- this closes #57 + +## v17.15.2 (date: 5.2.2024) + +- bugfix: venv activation script search performed after initialization + +## v17.15.1 (date: 5.2.2024) + +- bugfix: venv creation was missing `--system-site-packages` option, added +- bugfix: in venv creation, picking path from actual environment and then + using python there to create venv + +## v17.15.0 (date: 2.2.2024) + +- pull request from https://github.com/SoloJacobs/rcc relating to Windows + icacls usage. Thank you, Solomon Jacobs and Simon Meggle for bringing + this up. +- this closes #54 + +## v17.14.0 (date: 2.2.2024) + +- new experimental command `rcc holotree venv` to support python virtual + environments; this is still "work in progress" + +## v17.13.0 (date: 24.1.2024) + +- removing internal ECC experiment code (since it never get proper support) +- this should also remove one security vulnerability (Terrapin) hopefully + +## v17.12.1 (date: 27.11.2023) + +- bugfix: removing duplicates and existing holotree from PATHs before adding + new items in PATH + +## v17.12.0 (date: 23.11.2023) + +- reverted changes done in v17.8.0 (git hash 8771d622443efae2aa04c2d8c85b5b5c2e7aa3d6) + +## v17.11.0 (date: 23.11.2023) + +- adding functionality to mark holotree space as EXTERNALLY-MANAGED (PEP 668) + +## v17.10.0 (date: 16.11.2023) + +- functionality to tell rcc to not manage anything relating to temporaray + directories (that is, something else is managing those) +- new environment variable `RCC_NO_TEMP_MANAGEMENT` and new command line flag + `--no-temp-management` to control above thing +- functionality to tell rcc to not manage anything relating to python .pyc + files (that is, something else is managing those) +- new environment variable `RCC_NO_PYC_MANAGEMENT` and new command line flag + `--no-pyc-management` to control above thing +- added diagnostics warnings when above environment variables are set + +## v17.9.0 (date: 15.11.2023) + +- rcc is now checking if newer released versions are available, and adds + notification into stderr if not using that version + +## v17.8.1 (date: 14.11.2023) + +- bug fix: made check of users sharing `ROBOCORP_HOME` case insenstive +- added note on `ROBOCORP_HOME` permissions into documentation +- also `journal.run` has event when multiple users share same home + +## v17.8.0 (date: 14.11.2023) REVERTED + +- expanded documentation on `rcc robot dependencies` command +- added warning when developer declared dependencies file is missing, and + when environment configuration drift is shown +- added diagnostics to detect missing dependencies drift file + +## v17.7.3 (date: 14.11.2023) + +- changed subprocess monitoring from 200ms to 550ms, since on slower machines, + that 200ms causes too much load (experiment; might need to change later again) + +## v17.7.2 (date: 8.11.2023) + +- documentation updates on maintenance, and vocabulary, etc. + +## v17.7.1 (date: 8.11.2023) + +- bugfix: changed used holotree space tracking so, that it is visible to + everybody on file level + +## v17.7.0 (date: 8.11.2023) INCOMPLETE + +- added simple tracking of used holotree spaces +- added "Last used" and "Use count" to holotree space listings + +## v17.6.1 (date: 6.11.2023) WORK IN PROGRESS + +- removed experimental `SSL_CERT_DIR` as environment variable, since it might + actually be confusing to have there (but diagnostics will remain) +- removed duplicate work on checking catalog integrity which was called + during holotree restore + +## v17.6.0 (date: 2.11.2023) WORK IN PROGRESS + +- replaced trollhash with simple relocation detection algorithm and remove + trollhash from codebase + +## v17.5.0 (date: 30.10.2023) + +- added `SSL_CERT_DIR` and `NODE_EXTRA_CA_CERTS` as environment variables + when there is certificate bundle available +- also added diagnostics of those environment variables (plus others) +- minor documentation fixes +- tutorial: example of easy robot run + +## v17.4.2 (date: 25.10.2023) + +- minor fix: rcc point of view now has version number in it +- new `--anything` flag to allow adding to command line something unique or + note worthy about that specific line (had no effect what so ever) +- technical: updated some go module dependencies + +## v17.4.1 (date: 23.10.2023) WORK IN PROGRESS + +- verifying that tlsexport bundle can imported into certificate pool +- using system certificate store as base (if available), and updating + certificates there by default +- fix on conda.yaml merging on pip options case +- peeking `--debug` and `--trace` flags for preview of verbosity state + +## v17.4.0 (date: 23.10.2023) WORK IN PROGRESS + +- new subcommand, `rcc configuration tlsexport`, to export TLS certificates + from given set of secure and unsecure URLs +- now tlsprobe reports fingerprint using sha256 from raw certificate, not + just plain signature + +## v17.3.1 (date: 18.10.2023) + +- minor fix: now used micromamba version number is stored in separate asset + file, to keep things in sync between build scripts and rcc binary + +## v17.3.0 (date: 16.10.2023) WORK IN PROGRESS + +- embedded micromamba inside rcc executable +- removed micromamba download support since it extract all the way +- removing support for arm64 architectures (linux, mac, windows) since + embedded micromamba is not available on those architectures + +## v17.2.0 (date: 12.10.2023) + +- micromamba upgrade to v1.5.1 + +## v17.1.3 (date: 12.10.2023) + +- fix: made used environment configuration visible on progress entry and + also noted once on first unique contact + +## v17.1.2 (date: 11.10.2023) + +- bugfix: Windows micromamba activation failures +- bugfix: operating system information was leaking process STDERR +- added operating system information to speed test output + +## v17.1.1 (date: 11.10.2023) UNSTABLE + +- bugfix: operating system information executed differently in windows +- added hostname and user name to diagnostic information + +## v17.1.0 (date: 10.10.2023) UNSTABLE + +- operating system infomation on diagnostics and progress items + +## v17.0.1 (date: 10.10.2023) UNSTABLE + +- early detection of `--warranty-voided` flag to allow init usage +- more functionality skipped when "warranty voided", so that rcc is more + read-only with that flag + +## v17.0.0 (date: 4.10.2023) UNSTABLE + +- MAJOR breaking change: removed interactive configuration command, since + Setup Utility now better covers that functionality +- MAJOR breaking change: holotree is now layered by default and `--layered` + option is gone +- few documentation updates + +## v16.9.0 (date: 3.10.2023) UNSTABLE + +- deterioration: added `--warranty-voided` mode to make system less robust but + faster (do not use this mode, unless you really do know what you are doing) + +## v16.8.0 (date: 2.10.2023) + +- improvement: quick diagnostics now has settings.yaml age visible as seconds +- added `RCC_REMOTE_ORIGIN` variable to diagnostics output +- deprecated interactive configuration, since Setup Utility should be used + +## v16.7.1 (date: 29.9.2023) + +- bugfix: added process blacklist to prevent old processes shown as child + processes in process tree (also recycled PIDs will become "grey listed") + and this bug was detected in Windows +- improvement: changed command WaitDelay from 15 seconds to 3 seconds + +## v16.7.0 (date: 27.9.2023) + +- refactored profile commands into one file +- added support for removing configuration profiles +- updated robot tests to test profile removal +- fix: added 3 second timeout to TLS checks + +## v16.6.0 (date: 22.9.2023) + +- internal probe becomes `rcc configuration tlsprobe` command +- tlsprobe output improvements (address and DNS resolution) +- sending metrics of `rcc.cli.run.failure` when automation exit code is + something else than zero + +## v16.5.0 (date: 21.9.2023) + +- new variables set into environments: `RC_DISABLE_SSL`, `WDM_SSL_VERIFY`, + `NODE_TLS_REJECT_UNAUTHORIZED`, and `RC_TLS_LEGACY_RENEGOTIATION_ALLOWED` +- new settings option `legacy-renegotiation-allowed` +- removed `automation-studio` from `autoupdates:` in settings.yaml file +- settings.yaml version number updated to `2023.09` +- added 5 second timeout to probe connections + +## v16.4.1 (date: 21.9.2023) INTERNAL + +- improve: refining TLS probe (added cipher suite) + +## v16.4.0 (date: 21.9.2023) INTERNAL + +- feature: internal TLS probe implementation + +## v16.3.1 (date: 20.9.2023) + +- bug fix: extracting big template failed +- now some Progress steps have CPUs also visible, in addition to worker count + +## v16.3.0 (date: 19.9.2023) + +- extended using "rcc point of view" messaging to environment building, + post-install and pre-run scripts +- holotree variables also now has "rcc point of view" visible +- changed robot tests to match "rcc point of view" changes +- highlighted Progress steps with cyan/green/red color (where available) + +## v16.2.2 (date: 13.9.2023) + +- bugfix: process tree 1 second delay to prevent "too fast" process snapshots + on Windows +- refactoring some unused code out of codebase + +## v16.2.1 (date: 12.9.2023) + +- bugfix: detecting and truncating process tree with too deep child structure + +## v16.2.0 (date: 11.9.2023) + +- added relocations statistics on catalog listing (Relocate column) + +## v16.1.3 (date: 7.9.2023) + +- comment explaining why certain unsecure code forms is required when + TLS diagnostics are done (to explain Github CodeQL security warnings) + +## v16.1.2 (date: 6.9.2023) WORK IN PROGRESS + +- bug fix: allowing detection of lower levels of TLS versions +- minor improvement: diagnostics TLS firewall/proxy detection +- minor improvement: full certificate chain is now behind `--debug` flag + +## v16.1.1 (date: 5.9.2023) WORK IN PROGRESS + +- bug fix: added missing proxies to micromamba phase + +## v16.1.0 (date: 5.9.2023) WORK IN PROGRESS + +- Now advanced network diagnostics also have separate `tls-verify` configuration + to enable TLS verifications from custom addresses. + +## v16.0.1 (date: 5.9.2023) WORK IN PROGRESS + +- Added full signature chain "dump" in case where there is some kind of + certificate failure in TLS verification. Network diagnostics still. + +## v16.0.0 (date: 5.9.2023) WORK IN PROGRESS + +- Breaking change: there is new TLS verification in place in diagnostic, and + this can break some old setups because new warnings. + +## v15.3.0 (date: 30.8.2023) + +- added `journal.run` event log into artifacts directory +- tidying some golang dependencies and removing some unused files + +## v15.2.0 (date: 23.8.2023) + +- new strategy to manage micromamba, with its own directory based on version + number: `ROBOCORP_HOME/micromamba//` +- updated cleanup to manage micromamba location change +- bugfix: speedtest now does timing also in debug/trace mode (and some other + minor improvements) + +## v15.1.0 (date: 22.8.2023) + +- robot diagnostics now has indication of environment cacheability and also + warnings (category 5010) when something prevents caching +- lack of public cacheability is also visible on environment creation +- documentation updates and improvements +- minor improvements on process tree debugging + +## v15.0.0 (date: 21.8.2023) WORK IN PROGRESS + +- breaking change: dropped default value `rcc robot initialize --template` + option (now it must be given) +- breaking change: environment variable `RCC_VERBOSITY` with values "silent", + "debug", and "trace" now override CLI options +- bugfix, process tree detecting and printing +- added debug/trace logging into process baby sitter +- work in progress: detecting cacheable environment configurations +- micromamba upgrade back to v1.4.9 (next trial) + +## v14.15.4 (date: 17.8.2023) + +- micromamba downgraded to v1.4.2 due to argument change + +## v14.15.3 (date: 14.8.2023) + +- added error message on canary failures +- added one diagnostics detail to show if global shared holotree is enabled + +## v14.15.2 (date: 10.8.2023) + +- bugfix, now giving little bit more stack to process tree + +## v14.15.1 (date: 9.8.2023) BROKEN + +- bugfix on process tree ending up eating too much stack (stack overflow) + +## v14.15.0 (date: 3.8.2023) BROKEN + +- micromamba upgrade to v1.4.9 + +## v14.14.0 (date: 27.6.2023) + +- unless silenced, always show "rcc point of view" of success or failure of + actual main robot run, on point of robot process exit + +## v14.13.3 (date: 22.6.2023) + +- faster heartbeat for snapshotting subprocesses during robot run (200ms) +- added guiding text on "non-empty artifacts directory case" + +## v14.13.2 (date: 21.6.2023) + +- micromamba downgrade to v1.4.2, because micromamba bug in Windows + +## v14.13.1 (date: 20.6.2023) UNSTABLE + +- bugfix: fixing exit code masking by subprocess handling +- predicting rcc exit code made visible +- making robot run exit code more visible +- robot tests now use special settings.yaml to prevent template updates and + will only use internal templates for testing + +## v14.13.0 (date: 15.6.2023) UNSTABLE + +- improved listing of still running processes +- set process wait delay to 15 seconds after process has completed but has not + released it IO pipes yet + +## v14.12.0 (date: 13.6.2023) UNSTABLE + +- adding listing of still running processes after robot run +- upgrading github actions to use go v1.20.x +- bugfix: panic when using lockpids with nil value + +## v14.11.0 (date: 12.6.2023) UNSTABLE + +- added `--switch` option to profile import to immediately switch to imported + profile once it is successfully imported + +## v14.10.0 (date: 12.6.2023) UNSTABLE + +- saving separate info file for catalogs and holotrees (to speed up some + commands in future) +- added interrupt signal ignoring around robot run, so that robot can actually + react and respond to interrupt (and if send twice, then second interrupt + will actually interrupt rcc) + +## v14.9.2 (date: 8.6.2023) UNSTABLE + +- more cleaning up of dead code and data structures +- made it visible if artifactsDir already have files before run starts + +## v14.9.1 (date: 7.6.2023) UNSTABLE + +- micromamba upgrade to v1.4.3 + +## v14.9.0 (date: 7.6.2023) + +- added one user per `ROBOCORP_HOME` warnings +- added also diagnostics to warn about above issue +- full cleanup now also removes `rcccache.yaml` file +- removed "Robots" section from `rcccache.yaml` file + +## v14.8.2 (date: 6.6.2023) UNSTABLE + +- added missing golden yaml file saving on layers +- added worker count on second progress indicator +- reporting relative time ratios on setup/run balances +- fixed bug in buildstats, where it was using global variables (instead of "it") + +## v14.8.1 (date: 5.6.2023) UNSTABLE + +- added `RCC_HOLOTREE_SPACE_ROOT` to environment variables provided by rcc +- saving `rcc_plan.log` into intermediate layers as well (and it is now in + memory presentation while building environment) +- restoring partial environment from layers and skipping already available + layers (but still only behind `--layered` flag) +- layers add new Progress step to rcc, now total is 15 steps. Test changed + to match that. + +## v14.8.0 (date: 31.5.2023) UNSTABLE + +- support for separating layers and calculating their fingerprints +- showing fingerprints on build output and in timeline (still only visualization) +- added controlling flag `--layered` to enable layer handling +- added recording of layers if above flag is given + +## v14.7.0 (date: 15.5.2023) + +- adding logical layers on holotree installation (visible on timeline) + +## v14.6.0 (date: 4.5.2023) + +- adding `--quick` flag to diagnostics to filter out slow diagnostics +- for now, "slow diagnostics" are mostly network related checks, some of + subprocesses still get executed (like micromamba for example) + +## v14.5.0 (date: 3.5.2023) + +- subprocess exit codes are now visible, when subprocess fails (that is, when + exit code in non-zero) +- minor update on "custom templates" documentation + +## v14.4.1 (date: 19.4.2023) + +- MAJOR BREAKING CHANGES: + - under "spring cleaning" umbrella + - virtual environment and `pyvenv.cfg` support removed after realization + that holotree environments are not virtual environments, they are full + environments and then some, they can also be called soft-containers + - by trying to be virtual environment also caused bug in Windows, where + `site-pacakges` and `Scripts` directories could be polluting all other + environments as well, and that is why there is some cleanup in place now + - removed old, unused functionality, specially commands `rcc robot fix`, + `rcc robot libs`, and `rcc robot list` and their relating functionality + +## v14.4.0 (date: 19.4.2023) UNSTABLE + +- major breaking change: removed `rcc robot libs` command, since it is not + used in tooling, and if needed, needs better design + +## v14.3.0 (date: 19.4.2023) UNSTABLE + +- major breaking change: removed `rcc robot fix` command (just command, + internal functionality is still used by rcc) + +## v14.2.0 (date: 18.4.2023) UNSTABLE + +- cleanup functionality for "Scripts" and "site-packages" that are in wrong + place (due virtual environment bug fixed in v14.0.0) + +## v14.1.0 (date: 18.4.2023) UNSTABLE + +- major breaking change: removed `rcc robot list` command and history handling + support (this was old Lab requested functionality) + +## v14.0.0 (date: 17.4.2023) UNSTABLE + +- major breaking change: this will remove some old, now unwanted functionality +- this will be ongoing work for short while, making things unstable for now +- removal of "virtual environment" support (pyvenv.cfg), and `VIRTUAL_ENV` + variable is no longer available + +## v13.12.3 (date: 14.4.2023) + +- improvement: more clear messaging on hololib corruption +- fix: full cleanup will first remove catalogs and then hololib + +## v13.12.2 (date: 13.4.2023) + +- updating documentation around `robot.yaml` and its functionality + +## v13.12.1 (date: 12.4.2023) + +- fix: added .poetry to list of ignored paths + +## v13.12.0 (date: 12.4.2023) + +- micromamba upgrade to v1.4.2 +- test change: removed test that can fail because of probabilistic feature + on some metric updates (which cause rcccache.yaml not to be written at all) + +## v13.11.0 (date: 5.4.2023) + +- tighter permissions restrictions of rcc.yaml and rcccache.yaml using + os.Chmod, so probably works on Mac and Linux, but Windows is uncertain + +## v13.10.1 (date: 20.3.2023) + +- diagnostics: minor wording change (removing "toplevel" references) +- documentation: some refinements and additions + +## v13.10.0 (date: 16.3.2023) + +- documentation: added holotree maintenance documentation +- documentation: added vocabulary/glossary for rcc used terms +- both above also as `rcc docs` subcommands + +## v13.9.2 (date: 9.3.2023) DOCUMENTATION + +- bugfix: updated toc.py to generate improved table of contents + +## v13.9.1 (date: 9.3.2023) + +- bugfix: zip verification failed when Windows uses backslashes in paths +- adding diagnostics around `ROBOCORP_HOME` location and robots +- minor documentation updates in relation to `ROBOCORP_HOME` usage + +## v13.9.0 (date: 8.3.2023) + +- added initial support for verifying that holotree imported zip structure shape + matches expected hololib catalog patterns (behind `--strict` flag, for now) + +## v13.8.0 (date: 7.3.2023) + +- new `--export` option to `rcc holotree prebuild` command, to enable direct + export to given hololib.zip filename of new, successfully build catalogs +- bugfix: catalog was exported before its content, which would make it so, that + catalog is present before its parts + +## v13.7.1 (date: 27.2.2023) + +- added missing `RCC_REMOTE_AUTHORIZATION` variable handling to rcc and passing + that variable to rccremote on pull requests + +## v13.7.0 (date: 27.2.2023) + +- troubleshooting documentation added as `rcc man troubleshooting` command +- consolidated and streamlined documentation commands into fewer source files +- added robot tests for documentation commands + +## v13.6.5 (date: 23.2.2023) + +- dependabot raised update on golang.org/x/text module (upgraded) +- security related dependency upgrade + +## v13.6.4 (date: 23.2.2023) DOCUMENTATION + +- documentation updates on netdiagnostics and troubleshooting + +## v13.6.3 (date: 20.2.2023) + +- change: changed WorkGroup to not use buffers on incoming messages, since it + will be more deterministic + +## v13.6.2 (date: 16.2.2023) UNSTABLE + +- bugfix: changed WaitGroup to WorkGroup (self implemented work synchronization) + +## v13.6.1 (date: 15.2.2023) + +- experiment: using probability to run some of maintenance functions and + making rcc little bit faster depending on chance + +## v13.6.0 (date: 10.2.2023) + +- upgrade: upgrading github actions and also using newer golang and python there + +## v13.5.8 (date: 10.2.2023) + +- bugfix: holotree delete --space option was always set + +## v13.5.7 (date: 10.2.2023) + +- bugfix: holotree delete and plan were doing too many calls to find same + environments (which mean they were really slow) +- some name refactorings to clarify intent of functions + +## v13.5.6 (date: 8.2.2023) + +- bugfix: create missing folders while creating and writing some files +- improvement: added optional top N biggest files sizes on catalog listing + +## v13.5.5 (date: 7.2.2023) + +- bugfix: contain output of checking long path support on Windows +- improvement: adding more structure to holotree pull timeline + +## v13.5.4 (date: 6.2.2023) UNSTABLE + +- bugfix: file syncing on pull commands + +## v13.5.3 (date: 6.2.2023) UNSTABLE + +- rccremote server zip file managementent improvements + +## v13.5.2 (date: 2.2.2023) + +- rccremote server timeout adjustments to much longer times (experimental) + +## v13.5.1 (date: 2.2.2023) + +- fixing progress counter on timeline output +- timeline output clarifications on hololib pull step + +## v13.5.0 (date: 2.2.2023) + +- support for pulling hololib catalogs as part of normal holotree environment + creation process (new Progress step). + +## v13.4.3 (date: 1.2.2023) + +- bugfix: shortcutting to file resource on cloud.ReadFile if actual exiting + file is given as resource link. + +## v13.4.2 (date: 31.1.2023) UNSTABLE + +- fixed broken holotree pull command, and made it allow pulling from plain + http sources + +## v13.4.1 (date: 30.1.2023) UNSTABLE + +- prebuild now needs shared holotree to be enabled before building +- prebuilds can now be forced for full rebuilds + +## v13.4.0 (date: 30.1.2023) UNSTABLE + +- peercc is renamed to rccremote, and peercc package renamed to remotree + +## v13.3.0 (date: 27.1.2023) UNSTABLE + +- feature: command for prebuilding environments (from files or from URLs) +- improvement: rcc version visible in "Toplevel" command list +- added support for "cloud.ReadFile" functionality +- bugfix: wrapped os.TempDir functionality to ensure directory exists + +## v13.2.0 (date: 24.1.2023) + +- feature: peercc force pulling holotree catalog from other remote peercc +- self pulling should be prevented and so protect self loops +- new settings version, 2023.01 with autoupdates for lab removed and + setup-utility added + +## v13.1.2 (date: 23.1.2023) + +- improvement: netdiagnostics with `--trace` flag will now list response + header information + +## v13.1.1 (date: 20.1.2023) UNSTABLE + +- fix: netdiagnostics configuration flag change (now it is `--checks filename`) + +## v13.1.0 (date: 19.1.2023) UNSTABLE + +- feature: more network related configurable diagnostics + +## v13.0.1 (date: 17.1.2023) + +- bugfix: diagnostics of ignoreFiles was not using paths correctly + +## v13.0.0 (date: 17.1.2023) + +- major breaking change: various robot unzipping method now flatten directory + tree so that paths used in robots are shorter and not so easily cause + problems and confusion + +## v12.3.1 (date: 16.1.2023) MAJOR BREAK + +- bugfix: unwrap worked wrongly in case of "." dir prefix + +## v12.3.0 (date: 13.1.2023) BUGGY MAJOR BREAK + +- feature: unwrap command now removes extra middle parts of file paths when + unzipping robot.zip files + +## v12.2.0 (date: 11.1.2023) + +- micromamba upgrade to v1.1.0 + +## v12.1.2 (date: 11.1.2023) + +- bugfix: parallel long path checks failed because not unique path was used, + added pid as part of that long path (just Windows), this closes #45 + +## v12.1.1 (date: 4.1.2023) + +- bugfix: adding more info when zip extraction fails + +## v12.1.0 (date: 3.1.2023) + +- feature: on assistant runs, if CR does not give artifact URL for uploading + artifacts, then it is now considered as disabled functionality (not error) + and no artifacts are pushed into cloud + +## v12.0.1 (date: 3.1.2023) + +- added diagnostics on loading ignoreFiles entry, which does not contain + any patterns in it +- updated documentation about `ignoreFiles:` in recipes, with hopefully + better explanation of how it should be used + +## v12.0.0 (date: 29.12.2022) UNSTABLE + +- adding "grace period" in "token time" calculations, and this is breaking + change, because token time calculation changes, and management of grace + period is user/app responsibility (but there is default value) and tokens + also will now have minimum period +- bugfix: when broken catalog was loaded, catalog listing failed + +## v11.36.5 (date: 28.12.2022) + +- fix: added more explanation to network diagnostics reporting, explaining + what actual successful check option did + +## v11.36.4 (date: 22.12.2022) + +- bugfix: added missing lock protections around importing and pulling holotrees + +## v11.36.3 (date: 21.12.2022) + +- improvement: added more color and changed wording on lock wait messages + +## v11.36.2 (date: 21.12.2022) + +- improvement: when there is longer lock wait, possible lock holders are listed + on console output and in timeline + +## v11.36.1 (date: 20.12.2022) + +- bugfix: diagnostics fail on new machine to touch lock files when directory + does not exist, this closes #43 +- bugfix: stale lock pid files are shown too often, this closes #42 +- diagnostics will now show hopefully more human friendly message when active + locks are detected +- added more runtime.Gosched calls to enable background go routines to have + chance to finish before application closes + +## v11.36.0 (date: 15.12.2022) + +- added category field into diagnostics JSON output, to support applications + to report better diagnostics messages + +## v11.35.7 (date: 15.12.2022) + +- this v11.35.x series adds new "peercc" executable and new holotree pull + subcommand to rcc; these are work in progress, and not ready for production + work yet; do not use, unless you know what you are doing +- added automatic import of delta environment update data +- tech: moved TryRemove, TryRemoveAll, and TryRename to pathlib +- tech: some zipper log verbosity was moved from Debug to Trace level + +## v11.35.6 (date: 14.12.2022) UNSTABLE + +- bug fix: ignoring dotfiles and directories in "pids" directory +- added new `rcc holotree pull` command to do delta environment update request + to peercc (still incomplete, does not do automatic import of content) +- on delta export zip, catalog will now come as last part of that zip from wire +- added set membership map functionality (to make faster membership checks on + bigger member sets) +- more failed parts of PoC removed (export specification and support functions) + +## v11.35.5 (date: 9.12.2022) UNSTABLE + +- fixed bug where last line of request was missing +- trying to fix CodeQL security warning (user input was already filtered based + on known set of values, but analyzer did not understand that) + +## v11.35.4 (date: 8.12.2022) UNSTABLE + +- removing failed parts of PoC +- added handler for streaming of requested catalog and missing parts +- made robot tests to automatically disconnect from shared holotree + +## v11.35.3 (date: 7.12.2022) UNSTABLE + +- replaced deprecated "ioutil" with suitable functions elsewhere, thank you + for Juneezee (Eng Zer Jun) for pointing these out in PR#40 +- added ComSpec, LANG and SHELL from environment into diagnostics output + +## v11.35.2 (date: 7.12.2022) UNSTABLE + +- next try to fix ruby support in GHA + +## v11.35.1 (date: 7.12.2022) UNSTABLE + +- github actions updated to use ruby 2.7 (github stopped supporting used 2.5) + +## v11.35.0 (date: 7.12.2022) UNSTABLE + +- starting new PoC on topic of "peer rcc" +- export specification simplification: now supports exactly one "wants" value + and it is not list anymore, but just plain and simple string +- added new "set" operations to support PoC functionality (generics) +- one part of PoC failed, but code is still there + +## v11.34.0 (date: 29.11.2022) + +- compiling rcc for arm64 architectures (linux, mac, windows) + +## v11.33.2 (date: 24.11.2022) + +- configuration diagnostics now measure and report time it takes to resolve + set of hostnames found from settings files + +## v11.33.1 (date: 23.11.2022) UNSTABLE + +- some additional timeline markers on assistant runs + +## v11.33.0 (date: 18.11.2022) UNSTABLE + +- feature: holotree delta export (for missing things only) +- changes normal holotree export command to support ".hld" files + +## v11.32.6 (date: 15.11.2022) + +- bugfix: from now on, lock pid files will only give diagnostic "warning" when + they are less than 12 hours old, after that they will be labeled as "stale" + and will still be visible in diagnostics, but on "ok" level + +## v11.32.5 (date: 15.11.2022) + +- cleanup: removing dead code that was not used anymore + +## v11.32.4 (date: 15.11.2022) + +- cleanup: removing old run minutes and stat lines (holotree stats cover those) + +## v11.32.3 (date: 14.11.2022) + +- holotree statistics are now part of human readable diagnostics when there + is 5 or more entries in statistics (but not available in JSON output) +- added cumulative statistics section into output +- bugfix: calculation mistakes in case of missing steps +- bugfix: detecting successful build + +## v11.32.2 (date: 11.11.2022) + +- added week limitation option for holotree statistics command +- added filter flags for assistants, robots, prepares, and variables for + holotree statistics command + +## v11.32.1 (date: 11.11.2022) UNSTABLE + +- feature: command to show local holotree environment build statistics + +## v11.32.0 (date: 10.11.2022) UNSTABLE + +- feature: local recording of holotree environment build statistics events +- moved journals to `ROBOCORP_HOME/journals` directory (and build stats will + be part of those journals) +- added pre run scripts to timeline + +## v11.31.2 (date: 8.11.2022) + +- bugfix: removing path separators from user name on lock pid files + +## v11.31.1 (date: 8.11.2022) + +- bugfix: changed lock pid filename not to contain extra dots +- added more info on pending lock files diagnostics check +- more debug information on Windows locking behaviour + +## v11.31.0 (date: 7.11.2022) + +- micromamba upgrade to v1.0.0 + +## v11.30.1 (date: 7.11.2022) + +- bugfix: added more checks around shared holotree enabling and using +- bugfix: make all lockfiles readable and writable by all +- added "diagnostics" command to toplevel commands + +## v11.30.0 (date: 2.11.2022) + +- added warning when vulnerable openssl is installed in environment + +## v11.29.1 (date: 26.10.2022) UNSTABLE + +- robot tests for unmanaged holotree spaces (revealed bugs) +- bugfix: correct checking of unmanaged space conflicts (on creation) + +## v11.29.0 (date: 25.10.2022) BROKEN + +- started adding support for unmanaged holotree spaces, to enable IT managed + holotree spaces (rcc will create them once, but integrity check are not + done when unmanaged spaces are used) +- bugfix: removing also .lck files when removing space + +## v11.28.3 (date: 19.10.2022) + +- added configuration diagnostic reporting on locking pids information + +## v11.28.2 (date: 19.10.2022) + +- made lock wait messages little more descriptive and added more of them +- added "pids" folder to keep track who is holding locks (just information) + +## v11.28.1 (date: 12.10.2022) + +- bugfix: direct initializing robot did not update templates + +## v11.28.0 (date: 5.10.2022) + +- micromamba upgrade to v0.27.0 +- refactored version micromamba version numbering into one place +- added used pip and micromamba versions in progress messages +- BUGFIX: now explicitely using environment python to run pip commands + (using `python -m pip install ...` form instead old `pip install` form) + +## v11.27.3 (date: 29.9.2022) + +- fix: adding more "plan analyzer" identifiers to its output +- fix: adding detection to "failed to build" messages +- fix: added json output to new robot creation to cloud + +## v11.27.2 (date: 27.9.2022) + +- improving plan analyzer with more rules to show messages + +## v11.27.1 (date: 26.9.2022) + +- fixing CodeQL security warning about allocation overflow + +## v11.27.0 (date: 23.9.2022) + +- support for analyzing installation plans and their challenges and show it + online, or afterwards +- analysis is visible in `rcc holotree plan` command and also in `pip` + phase in environment creation + +## v11.26.6 (date: 19.9.2022) + +- try to upgraded cobra and viper dependencies, to get remove security warnings + given by AWS container scanner tooling +- upgrade to use github.com/spf13/cobra v1.5.0 +- upgrade to use github.com/spf13/viper v1.13.0 +- upgrade to use gopkg.in/square/go-jose.v2 v2.6.0 + +## v11.26.5 (date: 16.9.2022) + +- added architecture/platform metric with same interval as timezone metrics +- `docs/history.md` updated with v11 information so far +- `docs/troubleshooting.md` updated with additional points + +## v11.26.4 (date: 14.9.2022) + +- new `docs/troubleshooting.md` document added +- new `docs/history.md` document added +- updated `scripts/toc.py` with new documents and minor improvement + +## v11.26.3 (date: 12.9.2022) + +- bugfix: moved "ht.lck" inside holotree location, and renamed it to be + `global.lck` file. +- added environment variable `SSL_CERT_FILE` to point into certificate bundle + if one is provided by profile +- documentation updates + +## v11.26.2 (date: 8.9.2022) + +- converted assets to embedded resources (golang builtin embed module) +- go-bindata is not used anymore (replaced by "embed") + +## v11.26.1 (date: 7.9.2022) + +- minor documentation improvement, highlighting configuration settings help, + that plain commands are showing vanilla rcc setting by default. + +## v11.26.0 (date: 7.9.2022) + +- experiment: pyvenv.cfg file written into created holotree before lifting +- update: cloud-linking in setting.yaml now points to new default location: + https://cloud.robocorp.com/link/ +- bugfix: settings.yaml version updated to 2022.09 (because options section) + +## v11.25.1 (date: 6.9.2022) + +- fix: symbolic link restoration, when target is actually non-symlink + +## v11.25.0 (date: 5.9.2022) + +- flag to show identity.yaml (conda.yaml) in holotree catalogs listing + and functionality then just show it as part of output, both human readable + and machine readable (JSON) + +## v11.24.0 (date: 2.9.2022) + +- refactoring some utility functions to more common locations +- adding rcc and micromamba binary locations to diagnostics +- added `RCC_EXE` environment variable available for robots +- added `RCC_NO_BUILD` environment variable support (in addition to + previous settings options and CLI flag; see v11.19.0) +- some documentation updates +- added support for toplevel `--version` option + +## v11.23.0 (date: 2.9.2022) + +- added unused option to holotree catalog removal command +- added maintenance related robot test suite +- minor documentation updates + +## v11.22.1 (date: 1.9.2022) BROKEN + +- fix: using wrong file for age calculation on holotree catalogs +- fix: holotree check failed to recover on corrupted files; now failure + leads to removal of broken file +- fix: empty hololib directories are now removed on holotree check + +## v11.22.0 (date: 31.8.2022) + +- new command `rcc holotree remove` added, and this will remove catalogs + from holotree library (hololib) +- added repeat count to holotree check command (used also from remove command) + +## v11.21.0 (date: 30.8.2022) + +- added support to tracking when catalog blueprints are used +- if there is no tracking info on existing catalog, first reporting will + reset it to zero (and report it as -1) +- added catalog age in days, and days since last used to catalog listing +- fixed bug on shared hololib location on catalog listing + +## v11.20.0 (date: 26.8.2022) + +- feature: allow holotree exporting using robot.yaml file. + +## v11.19.1 (date: 25.8.2022) + +- bug: empty entry on ignoreFiles caused unclear error +- fix: now empty entries are diagnosed and noted +- fix: also non-existing ignore files are diagnosed + +## v11.19.0 (date: 24.8.2022) + +- new global flag `--no-build` which prevents building environments, and + only allows using previously cached, prebuild or imported holotrees +- there is also "no-build" option in "settings.yaml" options section +- added "no-build" information to diagnostics output + +## v11.18.0 (date: 23.8.2022) + +- new cleanup option `--downloads` to remove downloads caches (conda, pip, + and templates) +- change: now conda pkgs is cleaned up also in quick cleanup (which now + includes all "downloads" cleanups) +- robot cache is now part of full cleanup +- run commands now cleanup their temp folders immediately + +## v11.17.2 (date: 19.8.2022) + +- bugfix: adding missing symbolic link handling of files and directories +- hololib catalogs now have rcc version information included +- added timeout to account deletion, to speed up unit tests + +## v11.17.1 (date: 18.8.2022) UNSTABLE + +- fix continued: adding missing symbolic link handling of files and directories + +## v11.17.0 (date: 17.8.2022) UNSTABLE + +- fix started: adding missing symbolic link handling of files and directories +- this will be UNSTABLE, work in progress, for now + +## v11.16.0 (date: 16.8.2022) + +- micromamba upgrade to v0.25.1 +- template upgrade of python to 3.9.13 +- template upgrade of pip to 22.1.2 +- template upgrade of rpaframework to 15.6.0 +- upgraded tests to match above version changes and their effects + +## v11.15.4 (date: 13.7.2022) + +- go-bindata was accidentally removed, adding it back + +## v11.15.3 (date: 13.7.2022) BROKEN + +- refactoring module dependencies to help reusing parts of rcc in other apps + +## v11.15.2 (date: 11.7.2022) + +- fixed table of contents links to match Github generated ones +- also tried to make toc.py more OS neutral (was failing on Windows) + +## v11.15.1 (date: 8.7.2022) + +- added "old school CI" recipe into documentation + +## v11.15.0 (date: 7.7.2022) + +- micromamba upgrade to v0.24.0 + +## v11.14.5 (date: 22.6.2022) + +- added `--once` flag to holotree shared enabling, in cases where costly + sharing is required only once + +## v11.14.4 (date: 15.6.2022) + +- holotree share enabling now uses "icals" in Windows to set default properties +- added marker file "shared.yes" when shared has been executed + +## v11.14.3 (date: 9.6.2022) + +- upgraded rcc to be build using go v1.18.x + +## v11.14.2 (date: 8.6.2022) + +- retry on fixing codeql-analysis problem + +## v11.14.1 (date: 8.6.2022) + +- fixing codeql-analysis settings and problems +- no codeql analysis for ruby or python in this repo + +## v11.14.0 (date: 7.6.2022) + +- experimenting on setting `VIRTUAL_ENV` environment variable to point into + environment rcc created environment +- made OS and architecture visible in rcc "Progress 2" marker + +## v11.13.0 (date: 19.5.2022) + +- new shared holotree should now be effective +- some instructions on recipes for enabling shared holotree +- micromamba upgrade to v0.23.2 + +## v11.12.9 (date: 17.5.2022) UNSTABLE + +- bugfix: effective user id did not work on windows, removing it for all OSs +- diagnostics now has true/false flag to indicated shared/private holotrees + +## v11.12.8 (date: 16.5.2022) UNSTABLE + +- bugfix: making shared directories shared only when they really are +- new command `rcc holotree shared --enable` to enable shared holotrees + in specific machine +- command `rcc holotree init` is now for normal users after shared command + +## v11.12.7 (date: 12.5.2022) UNSTABLE + +- micromamba upgrade to v0.23.1 +- added checks for hololib shared locations mode requirements + +## v11.12.6 (date: 10.5.2022) UNSTABLE + +- bugfix: added additional directory for hololib, since it helps mounting + on servers +- one recipe addition, for idea generation ... + +## v11.12.5 (date: 9.5.2022) UNSTABLE + +- micromamba upgrade to v0.23.0 + +## v11.12.4 (date: 9.5.2022) + +- bugfix: rcc task script could not find any task (reason: internal quoting) +- this closes #32 + +## v11.12.3 (date: 5.5.2022) UNSTABLE + +- Reverted the change in v11.12.2 based on further testing. + +## v11.12.2 (date: 5.5.2022) UNSTABLE + +- legacyfix: Adding `x-` prefix to custom header, due to some enterprise network proxies stripping headers. + +## v11.12.1 (date: 29.4.2022) + +- bugfix: duplicate devTask note when error needs to be shown + +## v11.12.0 (date: 28.4.2022) + +- `rcc holotree import` now supports URL imports + +## v11.11.2 (date: 26.4.2022) + +- more documentation update for devTasks + +## v11.11.1 (date: 26.4.2022) + +- bugfix: added v12 indicator in new form of holotree catalogs (to separate + from old ones) to allow old and new versions to co-exist +- documentation update for devTasks and ToC + +## v11.11.0 (date: 25.4.2022) + +- in addition to normal tasks, now robot.yaml can also contain devTasks +- it is activated with flag `--dev` and only available in task run command + +## v11.10.7 (date: 22.4.2022) + +- bugfix/retry: lock files are now marked as shared files (actually this + will not work on Windows on multi-user setup) +- changed robot test setup cleanup + +## v11.10.6 (date: 21.4.2022) + +- bugfix: lock files are now marked as shared files +- Rakefile: using `go install` for bindata from now on + +## v11.10.5 (date: 20.4.2022) + +- settings certificates section now has full path to CA bundle if available + +## v11.10.4 (date: 20.4.2022) + +- different preRunScripts for different operating systems +- acceptable differentiation patterns are: amd64/arm64/darwin/windows/linux + +## v11.10.3 (date: 19.4.2022) + +- fixed panic when settings.yaml is broken, now it will be blunt fatal failure +- fixed search path problem with preRunScripts (now robot.yaml PATH is used) +- removed direct mentions of Robocorp App (old name) + +## v11.10.2 (date: 13.4.2022) UNSTABLE + +- made presence of hololib.zip more visible on environment creation +- test changed so that new holotree relocation can be tested + +## v11.10.1 (date: 12.4.2022) UNSTABLE + +- adding support for user identity in relocation +- added holotree locations into diagnostics +- usage of hololib.zip now has debug entry in log files + +## v11.10.0 (date: 12.4.2022) UNSTABLE + +- work in progess: holotree relocation revisiting started +- now holotree library and catalog can be on shared location + +## v11.9.16 (date: 7.4.2022) + +- took pull request with documentation fixes + +## v11.9.15 (date: 6.4.2022) + +- improved table of contents with more clearer numbering + +## v11.9.14 (date: 6.4.2022) + +- improved table of contents with numbering + +## v11.9.13 (date: 6.4.2022) + +- added new script for generating table of contents for docs +- generated first table of contents as `docs/README.md` file + +## v11.9.12 (date: 6.4.2022) + +- added new `rcc man profiles` documentation command +- more documentation updates +- `Robocorp Cloud` to `Robocorp Control Room` related documentation changes + +## v11.9.11 (date: 5.4.2022) + +- documentation updates + +## v11.9.10 (date: 4.4.2022) + +- added current profile in JSON response from configuration switch command +- fixing bugs and typos in code and texts + +## v11.9.9 (date: 31.3.2022) + +- updated interactive create with `--task` option alternative +- updated run error message with `--task` option instructions +- this closes #28 +- updated `recipes.md` with python conversion instructions + +## v11.9.8 (date: 29.3.2022) + +- updated profile documentation +- added integrity check on hololib to space extraction +- more robot tests added +- fixed ssl-no-revoke bug (found thru new robot tests) + +## v11.9.7 (date: 28.3.2022) + +- profiles should now be good enough to start testing them +- interactive configuration now has instructions for next steps (kind of + scripted but not automated; copy-paste instructions) +- added placeholder `docs/profile_configuration.md` for future documentation +- settings.yaml now has Automation Studio autoupdate URL +- added `robot_tests/profiles.robot` to test new functionality + +## v11.9.6 (date: 25.3.2022) UNSTABLE + +- adding more setting options and environment variables +- added support for CA-bundles in pem format + +## v11.9.5 (date: 23.3.2022) UNSTABLE + +- refactoring variables exporting into one place +- adding `PIP_CONFIG_FILE`, `HTTP_PROXY`, and `HTTPS_PROXY` variables into + conda environment if they are configured + +## v11.9.4 (date: 22.3.2022) UNSTABLE + +- profile exporting now works + +## v11.9.3 (date: 22.3.2022) UNSTABLE + +- started to add real support for profile switching/importing +- some documentation updates + +## v11.9.2 (date: 18.3.2022) UNSTABLE + +- settings are now layered, so that partial custom settings.yaml also works +- settings now have flat API interface, that is used instead of direct access +- settings.yaml version upgrade with new fields (still incomplete) +- endpoints in settings are now a map and not separate structure anymore +- partial "demo" work on interactive configuration (work in progress) + +## v11.9.1 (date: 10.3.2022) UNSTABLE + +- added condarc and piprc to be asked from user as configuration options +- refactoring some wizard code to better support new functionality + +## v11.9.0 (date: 9.3.2022) UNSTABLE + +- new work started around network configuration topics and this will be + WIP (work in progress) for a while, and so it is marked as unstable +- added new command placeholders (no-op): `interactive configuration`, + `configuration export`, `configuration import`, and `configuration switch` + +## v11.8.0 (date: 8.3.2022) + +- added initial alpha support for pre-run scripts from robot.yaml and executed + right before actual task is run + +## v11.7.1 (date: 8.3.2022) + +- when timeline option is given, and operation fails, timeline was not shown + and this change now makes timeline happen before exit is done +- speed test now allows using debug flag to actually see what is going on + +## v11.7.0 (date: 8.3.2022) + +- micromamba update to version 0.22.0 + +## v11.6.6 (date: 7.3.2022) + +- JSON/YAML diagnostics is now ignoring anything that contains ".vscode" + +## v11.6.5 (date: 2.3.2022) + +- Still continuing GH#27 fixing issue where rcc finds executables outside of + holotree environment. + +## v11.6.4 (date: 23.2.2022) + +- GH#27 fixing issue where rcc finds executables outside of holotree + environment. +- this closes #27 + +## v11.6.3 (date: 10.1.2022) + +- more patterns added ("pypoetry" and "virtualenv") to be removed from PATH, + since they also can break isolation of our environments + +## v11.6.2 (date: 7.1.2022) + +- added "pyenv" and "venv" to patterns removed from PATH, since they can + break isolation of our environments + +## v11.6.1 (date: 7.1.2022) + +- fixing micromamba version number parsing + +## v11.6.0 (date: 7.12.2021) broken + +- micromamba update to version 0.19.0 +- now `artifactsDir` is explicitely created before robot execution + +## v11.5.5 (date: 2.11.2021) + +- bugfix: robot task format ignored artifacts directory, but now it uses it + +## v11.5.4 (date: 29.10.2021) + +- bugfix: path handling in robot wrap commands (now cross-platform) + +## v11.5.3 (date: 28.10.2021) broken + +- bugfix: path handling in robot wrap commands + +## v11.5.2 (date: 27.10.2021) + +- added `--json` option and output to catalogs listing +- bug fix: added missing file detection to holotree check + +## v11.5.1 (date: 26.10.2021) + +- adding holotree catalogs command to list available catalogs with more detail +- extending holotree list command to show all spaces reachable from hololib + catalogs including imported holotree spaces +- holotree delete should now also remove space elsewhere (based on imported + catalogs and their holotree locations) + +## v11.5.0 (date: 20.10.2021) + +- adding initial support for importing hololib.zips into local hololib catalog + +## v11.4.3 (date: 20.10.2021) + +- fixing bug where gzipped files in virtual holotree get accidentally + expanded when doing `--liveonly` environments +- added global `--workers` option to allow control of background worker count + +## v11.4.2 (date: 19.10.2021) + +- one more improvement on abstract score reporting (time is also scored) + +## v11.4.1 (date: 18.10.2021) + +- minor textual improvements on abstract score reporting + +## v11.4.0 (date: 18.10.2021) + +- new command `rcc configuration speedtest` which gives abstract score to both + network and filesystem speed +- some refactoring to enable above functionality + +## v11.3.6 (date: 13.10.2021) + +- bugfix: added retries to holotree file removal + +## v11.3.5 (date: 12.10.2021) + +- bugfix: added retries and better error message on holotree rename pattern + +## v11.3.4 (date: 12.10.2021) + +- new toplevel flag to turn on `--strict` environment handling, and for now + this make rcc to run `pip check` after environment install completes +- added timeout to metrics sending + +## v11.3.3 (date: 8.10.2021) + +- micromamba update to version 0.16.0 +- minor change on os.Stat usage in holotree functions +- changed minimum required worker count to 2 (was 4 previously) + +## v11.3.2 (date: 7.10.2021) + +- templates are removed when quick cleanup is requested +- bugfix: now debug and trace flags are also considered same as + `VERBOSE_ENVIRONMENT_BUILDING` environment variable +- bugfix: added some jupyter paths as skipped ingored ones in diagnostics +- added canary checks into diagnostics for pypi and conda repos + +## v11.3.1 (date: 5.10.2021) + +- using templates from templates.zip in addition to internal templates +- command `rcc holotree bootstrap` update to use templates.zip +- command `rcc interactive create` now uses template descriptions +- command `rcc robot init` now has `--json` flag to produce template list + as JSON +- settings.yaml updated to version 2021.10 + +## v11.3.0 (date: 4.10.2021) + +- update robot templates from cloud (not used yet, coming up in next versions) + +## v11.2.0 (date: 29.9.2021) + +- updated content to [recipes](/docs/recipes.md) about Holotree controls +- two new documentation commands, features and usecases, and corresponding + markdown documents in docs folder +- added env.json capability also into pure `conda.yaml` case in + `rcc holotree variables` command (bugfix) + +## v11.1.6 (date: 27.9.2021) + +### What to consider when upgrading from series 10 to series 11 of rcc? + +Major version break between rcc 10 and 11 was about removing the old base/live +way of managing environments (`rcc environment ...` commands). That had some +visible changes in rcc commands used. Here is a summary of those changes. + +- Compared to base/live based management of environments, holotree needs + a different mindset to work with. With the new holotree, users decide which + processes share the same working space and which receive their own space. + So, high level management of logical spaces has shifted from rcc to user + (or tools), where in base/live users did not have the option to do so. + Low level management is still rcc responsibility and based on "conda.yaml" + content. +- All `rcc environment` commands were removed or renamed, since this was + an old way of doing things. +- Old `rcc env hash` was renamed to `rcc holotree hash` and changed to show + holotree blueprint hash. +- Old `rcc env plan` was renamed to `rcc holotree plan` and changed to show + plan from given holotree space. +- Old `rcc env cleanup` was renamed to `rcc configuration cleanup` and + changed to work in a way that only holotree things are valid from now on. + This means that if you are using `rcc conf cleanup`, check help for changed + flags also. +- In general, the old `--stage` flag is gone, since it was base/live specific. +- Holotree related commands, including various run commands, now have default + values for the `--space` flag. So if no `--space` flag is given, that + defaults to `user` value, and the same space will be updated based + on requested environment specification. +- Output of some commands have changed, for example there are now more + "Progress" steps in rcc output. + +## v11.1.5 (date: 24.9.2021) + +- bugfix: performance profiling revealed bottleneck in windows, where calling + stat is expensive, so here is try to limit using it uneccessarily + +## v11.1.4 (date: 23.9.2021) + +- bugfix: adding concurrencty to catalog check +- performance profiling revealed bottleneck, where ensuring directory exist + was called too often, so now base directories are ensured only once per + rcc invocation +- adding more structure to timeline printout by indentation of blocks + +## v11.1.3 (date: 21.9.2021) + +- bugfix: changing performance thru auto-scaling workers based on number + of CPUs (minus one, but at least 4 workers) + +## v11.1.2 (date: 20.9.2021) + +- bugfix: removing duplicate file copy on holotree recording +- removed "new live" phrase from debug printouts +- made robot tests to check holotree integrity in some selected points + +## v11.1.1 (date: 17.9.2021) + +- bugfix: using rename in hololib file copy to make it more transactional +- progress indicator now has elapsed time since previous progress entry +- experimental upgrade to use go 1.17 on Github Actions + +## v11.1.0 (date: 16.9.2021) + +- BREAKING CHANGES, but now this may be considered stable(ish) +- micromamba update to version 0.15.3 +- added more robot tests and improved `rcc holotree plan` command + +## v11.0.8 (date: 15.9.2021) UNSTABLE + +- BREAKING CHANGES (ongoing work, see v11.0.0 for more details) +- showing correct `rcc_plan.log` and `identity.yaml` files on log +- reorganizing some common code away from conda module +- rpaframework upgrade to version 11.1.3 in templates + +## v11.0.7 (date: 14.9.2021) UNSTABLE + +- BREAKING CHANGES (ongoing work, see v11.0.0 for more details) +- changed progress indication to match holotree flow +- made log and telemetry waiting visible in timeline + +## v11.0.6 (date: 13.9.2021) UNSTABLE + +- BREAKING CHANGES (ongoing work, see v11.0.0 for more details) +- removing options from cleanup commands, since those are base/live specific + and not needed anymore (orphans, miniconda) +- removed dead code resulted from above + +## v11.0.5 (date: 10.9.2021) UNSTABLE + +- BREAKING CHANGES (ongoing work, see v11.0.0 for more details) +- removing conda environment build related code +- internal clone command was removed +- side note: there is trail of FIXME comments in code for future work + +## v11.0.4 (date: 9.9.2021) UNSTABLE + +- BREAKING CHANGES (ongoing work, see v11.0.0 for more details) +- replaced `rcc env plan` with new `rcc holotree plan`, which now shows + installation plans from holotree spaces +- now all env commands are removed, so also toplevel "env" command is gone +- added naive helper script, deadcode.py, to find dead code +- cleaned up some dead code branches + +## v11.0.3 (date: 8.9.2021) UNSTABLE + +- BREAKING CHANGES (ongoing work, see v11.0.0 for more details) +- removed commands "new", "delete", "list", and "variables" from `rcc env` + command set +- replaced `rcc env hash` with new `rcc holotree hash`, which now calculates + blueprint fingerprint hash similar way that env hash but differently + because holotree uses siphash algorithm + +## v11.0.2 (date: 8.9.2021) UNSTABLE + +- BREAKING CHANGES (ongoing work, see v11.0.0 for more details) +- technical work: cherry-picking changes from v10.10.0 into series 11 + +## v11.0.1 (date: 7.9.2021) UNSTABLE + +- BREAKING CHANGES (ongoing work, see v11.0.0 for more details) +- fixing robot tests + +## v11.0.0 (date: 6.9.2021) UNSTABLE + +- BREAKING CHANGES (ongoing work, small steps, considered unstable) and goal + is to remove old base/live environment handling and make holotree default + and only way to manage environments +- setting "user" as default space for all commands that need environments + +## v10.10.0 (date: 7.9.2021) + +- this is series 10 maitenance branch +- rcc config cleanup improvement, so that not partial cleanup is done on + holotree structure (on Windows, respecting locked environments) + +## v10.9.4 (date: 31.8.2021) + +- invalidating hololib catalogs with broken files in hololib + +## v10.9.3 (date: 31.8.2021) + +- added diagnostic warnings on `PLAYWRIGHT_BROWSERS_PATH`, `NODE_OPTIONS`, + and `NODE_PATH` environment variables when they are set + +## v10.9.2 (date: 30.8.2021) + +- bugfix: long running assistant run now updates access tokens correctly + +## v10.9.1 (date: 27.8.2021) + +- made problems in assistant heartbeats visible +- changed assistant heartbeat from 60s to 37s to prevent collision with + DNS TTL value + +## v10.9.0 (date: 25.8.2021) + +- added --quick option to `rcc config cleanup` command to provide + partial cleanup, but leave hololib and pkgs cache intact + +## v10.8.1 (date: 24.8.2021) + +- holotree check command now removes orphan hololib files +- environment creation metrics added on failure cases +- pip and micromamba exit codes now also in hex form +- minor error message fixes for Windows (colors) + +## v10.8.0 (date: 19.8.2021) + +- added holotree check command to verify holotree library integrity +- added "env cleanup" also as "config cleanup" +- minor go-routine schedule yield added (experiment) + +## v10.7.1 (date: 18.8.2021) + +- bugfix: trying to remove preformance hit on windows directory cleanup + +## v10.7.0 (date: 16.8.2021) + +- when environment creation is serialized, after short delay, rcc reports + that it is waiting to be able to contiue +- added \_\_MACOSX as ignored files/directories + +## v10.6.0 (date: 16.8.2021) + +- added possibility to also delete holotree space by providing controller + and space flags (for easier scripting) + +## v10.5.2 (date: 12.8.2021) + +- added once a day metric about timezone where rcc is executed + +## v10.5.1 (date: 11.8.2021) + +- improvements for detecting OS/architecture for multiple environment + configurations + +## v10.5.0 (date: 10.8.2021) + +- supporting multiple environment configurations to enable operating system + and architecture specific freeze files (within one robot project) + +## v10.4.5 (date: 10.8.2021) + +- bugfix: removing one more filesystem sync from holotree (Mac slowdown fix). + +## v10.4.4 (date: 9.8.2021) broken + +- bugfix: raising initial scaling factor to 16, so that there should always + be workers waiting + +## v10.4.3 (date: 9.8.2021) broken + +- bugfix: trying to fix Mac related slowing by removing file syncs on + holotree copy processes + +## v10.4.2 (date: 5.8.2021) broken + +- bugfix: scaling down holotree concurrency, since at least Mac file limits + are hit by current concurrency limit + +## v10.4.1 (date: 5.8.2021) + +- taking micromamba 0.15.2 into use + +## v10.4.0 (date: 5.8.2021) + +- bug fix: `rcc_activate.sh` were failing, when path to rcc has spaces in it + +## v10.3.3 (date: 29.6.2021) + +- updated tips, tricks, and recipes + +## v10.3.2 (date: 29.6.2021) + +- fix for missing artifact directory on runs + +## v10.3.1 (date: 29.6.2021) broken + +- cleaning up `rcc robot dependencies` and related code now that freeze is + actually implemented +- changed `--copy` to `--export` since it better describes the action +- removed `--bind` because copying freeze file from run is better way +- removed "ideal" conda.yaml printout, since runs now create artifact + on every run in new envrionments +- removed those robot diagnostics that are misguiding now when dependencies + are frozen +- updated rpaframework to version 10.3.0 in templates +- updated robot tests for rcc + +## v10.3.0 (date: 28.6.2021) + +- creating environment freeze YAML file into output directory on every run + +## v10.2.4 (date: 24.6.2021) + +- added `--bind` option to copy exact dependencies from `dependencies.yaml` + into `conda.yaml`, so that `conda.yaml` represents fixed dependencies + +## v10.2.3 (date: 24.6.2021) + +- added `dependencies.yaml` into robot diagnostics +- show ideal `conda.yaml` that matches `dependencies.yaml` +- fixed `--force` install on base/live environments + +## v10.2.2 (date: 23.6.2021) + +- adding `rcc robot dependencies` command for viewing desired execution + environment dependencies +- same view is now also shown in run context replacing `pip freeze` if + golden-ee.yaml exists in execution environment + +## v10.2.1 (date: 21.6.2021) + +- showing dependencies listing from environment before runs + +## v10.2.0 (date: 21.6.2021) + +- adding golden-ee.yaml document into holotree space (listing of components) + +## v10.1.1 (date: 18.6.2021) + +- taking micromamba 0.14.0 into use + +## v10.1.0 (date: 17.6.2021) + +- adding pager for `rcc man xxx` documents +- more trace printing on workflow setup +- added [D] and [T] markers for debug and trace level log entries +- when debug and trace log level is on, normal log entries are prefixed with [N] +- fixed rights problem in file `rcc_plan.log` + +## v10.0.0 (date: 15.6.2021) + +- removed lease support, this is major breaking change (if someone was using it) + +## v9.20.0 (date: 10.6.2021) + +- added `rcc task script` command for running anything inside robot environment + +## v9.19.4 (date: 10.6.2021) + +- added json format to `rcc holotree export` output formats +- added docs/recipes.md and also new command `rcc docs recipes` +- added links to README.md to internal documentation + +## v9.19.3 (date: 10.6.2021) + +- added support for getting list of events out +- fix: moved holotree changes messages to trace level + +## v9.19.2 (date: 9.6.2021) + +- added locking of holotree into environment restoring + +## v9.19.1 (date: 8.6.2021) + +- added locking of holotree into new environment building and recording + +## v9.19.0 (date: 8.6.2021) + +- added event journaling support (no user visible yet) +- added first event "space-used" in holotree restore operations (this enables + tracking of all places where environments are created) + +## v9.18.0 (date: 3.6.2021) + +- now using holotree location from catalog, so that catalog decides where + holotree is created (defaults to `ROBOCORP_HOME` but can be different) +- if hololib.zip exist, then `--space` flag must be given or run fails +- hololib.zip is now reported in robot diagnostics +- environment difference print is now (mostly) behind `--trace` flag +- if rcc is not interactive, color toggling on Windows is skipped +- micromamba download is now done "on demand" only +- added robot tests for hololib.zip workflow + +## v9.17.2 (date: 2.6.2021) + +- fixing broken tests, and taking account changed specifications + +## v9.17.1 (date: 2.6.2021) broken + +- adding supporting structures for zip based holotree runs [experimental] + +## v9.17.0 (date: 26.5.2021) + +- added `export` command to holotree [experimental] + +## v9.16.0 (date: 21.5.2021) + +- catalog extension based on operating system, architecture and directory + location + +## v9.15.1 (date: 21.5.2021) + +- added images as non-executable files +- run and testrun commands have new option `--no-outputs` which prevent + capture of stderr/stdout into files +- separated `--trace` and `--debug` flags from `micromamba` and `pip` verbosity + introduced in v9.12.0 (it is causing too much output and should be reserved + only for `RCC_VERBOSE_ENVIRONMENT_BUILDING` variable + +## v9.15.0 (date: 20.5.2021) + +- for `task run` and `task testrun` there is now possibility to give additional + arguments from commandline, by using `--` separator between normal rcc + arguments and those intended for executed robot +- rcc now considers "http://127.0.0.1" as special case that does not require + https + +## v9.14.0 (date: 19.5.2021) + +- added PYTHONPATH diagnostics validation +- added `--production` flag to diagnostics commands + +## v9.13.0 (date: 18.5.2021) + +- micromamba upgrade to version 0.13.1 +- activation script fix for windows environment + +## v9.12.1 (date: 18.5.2021) + +- new environment variable `ROBOCORP_OVERRIDE_SYSTEM_REQUIREMENTS` to make + skip those system requirements that some users are willing to try +- first such thing is "long path support" on some versions of Windows + +## v9.12.0 (date: 18.5.2021) + +- new environment variable `RCC_VERBOSE_ENVIRONMENT_BUILDING` to make + environment building more verbose +- with above variable and `--trace` or `--debug` flags, both micromamba + and pip are run with more verbosity + +## v9.11.3 (date: 12.5.2021) + +- adding error signaling on anywork background workers +- more work on improving slow parts of holotree +- fixed settings.yaml conda link (conda.anaconda.org reference) + +## v9.11.2 (date: 11.5.2021) + +- added query cache in front of slow "has blueprint" query (windows) +- more timeline entries added for timing purposes + +## v9.11.1 (date: 7.5.2021) + +- new get/robot capabilitySet added into rcc +- added User-Agent to rcc web requests + +## v9.11.0 (date: 6.5.2021) + +- started using new capabilitySet feature of cloud authorization +- added metric for run/robot authorization usage +- one minor typo fix with "terminal" word + +## v9.10.2 (date: 5.5.2021) + +- added metrics to see when there was catalog failure (pre-check related) +- added PYTHONDONTWRITEBYTECODE=x setting into rcc generated environments, + since this will pollute the cache (every compilation produces different file) + without much of benefits +- also added PYTHONPYCACHEPREFIX to point into temporary folder +- added `--space` flag to `rcc cloud prepare` command + +## v9.10.1 (date: 5.5.2021) + +- added check for all components owned by catalog, to verify that they all + are actually there +- added debug level logging on environment restoration operations +- added possibility to have line numbers on rcc produced log output (stderr) +- rcc log output (stderr) is now synchronized thru a channel +- made holotree command tree visible on toplevel listing + +## v9.10.0 (date: 4.5.2021) + +- refactoring code so that runs can be converted to holotree +- added `--space` option to runs so that they can use holotree +- holotree blueprint should now be unified form (same hash everywhere) +- holotree now co-exists with old implementation in backward compatible way + +## v9.9.21 (date: 4.5.2021) + +- documentation fix for toplevel config flag, closes #18 + +## v9.9.20 (date: 3.5.2021) + +- added blueprint subcommand to holotree hierarchy to query blueprint + existence in hololib + +## v9.9.19 (date: 29.4.2021) + +- refactoring to enable virtual holotree for --liveonly functionality +- NOTE: leased environments functionality will go away when holotree + goes mainstream (and plan for that is rcc series v10) + +## v9.9.18 (date: 28.4.2021) + +- some cleanup on code base +- changed autoupdate url for Robocorp Lab + +## v9.9.17 (date: 20.4.2021) + +- added environment, workspace, and robot support to holotree variables command +- also added some robot tests for holotree to verify functionality + +## v9.9.16 (date: 20.4.2021) + +- added support for deleting holotree controller spaces +- added holotree and hololib to full environment cleanup +- added required parameter to `rcc env delete` command also + +## v9.9.15 (date: 19.4.2021) + +- bugfix: locking while multiple rcc are doing parallel work should now + work better, and not corrupt configuration (so much) + +## v9.9.14 (date: 15.4.2021) + +- environment variables conda.yaml ordering fix (from robot.yaml first) +- task shell does not need task specified anymore + +## v9.9.13 (date: 15.4.2021) + +- fixing environment variables bug from below + +## v9.9.12 (date: 15.4.2021) + +- updated rpaframework to version 9.5.0 in templates +- added more timeline entries around holotree +- minor performance related changes for holotree +- removed default PYTHONPATH settings from "taskless" environment +- known, remaining bug: on "env variables" command, with robot without default + task and without task given in CLI, environment wont have PATH or PYTHONPATH + or robot details setup correctly + +## v9.9.11 (date: 13.4.2021) + +- added support for listing holotree controller spaces + +## v9.9.10 (date: 12.4.2021) + +- removed index.py utility, since better place is on other repo, and it + was mistake to put it here + +## v9.9.9 (date: 9.4.2021) + +- fixed index.py utility tool to work in correct repository + +## v9.9.8 (date: 9.4.2021) + +- skip environment bootstrap when there is no conda.yaml used +- added index.py utility tool for generating index.html for S3 + +## v9.9.7 (date: 8.4.2021) + +- now `rcc holotree bootstrap` can only download templates with `--quick` + flag, or otherwise also prepare environment based on that template + +## v9.9.6 (date: 8.4.2021) + +- holotree note: in this series 9, holotree will remain experimental and + will not be used for production yet +- added separate `holotree` subtree in command structure (it is not internal + anymore, but still hidden) +- partial implementations of holotree variables and bootstrap commands +- settings.yaml version 2021.04 update: now there is separate section + for templates +- profiling option `--pprof` is now global level option +- improved error message when rcc is not configured yet + +## v9.9.5 (date: 6.4.2021) + +- micromamba upgrade to version 0.9.2 + +## v9.9.4 (date: 6.4.2021) + +- fix for holotree change detection when switching blueprints + +## v9.9.3 (date: 1.4.2021) + +- added export/SET prefix to `rcc env variables` command +- updated README.md with patterns to version numbered releases +- known bug: holotree does not work correctly yet -- DO NOT USE + +## v9.9.2 (date: 1.4.2021) + +- more holotree integration work to get it more experimentable + +## v9.9.1 (date: 31.3.2021) + +- Github Actions upgrade to use Go 1.16 for rcc compilation + +## v9.9.0 (date: 31.3.2021) broken + +- added holotree as part of source code (but not as integrated part yet) +- added new internal command: holotree + +## v9.8.11 (date: 30.3.2021) + +- added Accept header to micromamba download command +- made some URL diagnostics optional, if they are left empty + +## v9.8.10 (date: 30.3.2021) + +- fix: no more panics when directly writing to settings.yaml + +## v9.8.9 (date: 29.3.2021) + +- added `cloud-ui` to settings.yaml + +## v9.8.8 (date: 29.3.2021) + +- mixed fixes and experiments edition +- ignoring empty variable names on environment dumps, closes #17 +- added some missing content types to web requests +- added experimental ephemeral ECC implementation +- more common timeline markers added +- will not list pip dependencies on assistant runs +- will not ask cloud for runtime authorization (bug fix) + +## v9.8.7 (date: 26.3.2021) + +- more finalization of settings.yaml change +- made micromamba less quiet on environment building +- secrets now have write access enabled in rcc authorization requests +- if merged conda.yaml files do not have names, merge result wont have either + +## v9.8.6 (date: 25.3.2021) + +- settings.yaml cleanup +- fixed robot tests for 9.8.5 template changes + +## v9.8.5 (date: 24.3.2021) + +- Robot templates updated: Rpaframework updated to v9.1.0 +- Robot templates updated: Improved task names +- Robot templates updated: Extended template has example of multiple tasks execution + +## v9.8.4 (date: 24.3.2021) + +- fix for pip made too silent on this v9.8.x series +- and also in failure cases, print out full installation plan + +## v9.8.3 (date: 24.3.2021) + +- can configure all rcc operations not to verify correct SSL certificate + (please note, doing this is insecure and allows man-in-the-middle attacks) +- applied reviewed changes to what is actually in settings.yaml file + +## v9.8.2 (date: 23.3.2021) + +- ALPHA level pre-release (do not use, unless you know what you are doing) +- reorganizing some code to allow better use of settings.yaml +- more values from settings.yaml are now used + +## v9.8.1 (date: 22.3.2021) + +- ALPHA level pre-release (do not use, unless you know what you are doing) +- now some parts of settings are used from settings.yaml +- settings.yaml is now critical part of rcc, so diagnostics also contains it +- also from now, problems in settings.yaml may make rcc to fail +- changed ephemeral key size to 2048, which should be good enough + +## v9.8.0 (date: 18.3.2021) + +- ALPHA level pre-release with settings.yaml (do not use, unless you know + what you are doing) +- started to moved some of hardcoded things into settings.yaml (not used yet) +- minor assistant upload fix, where one error case was not marked as error + +## v9.7.4 (date: 17.3.2021) + +- typo fix pull request from jaukia +- added micromamba --no-rc flag + +## v9.7.3 (date: 16.3.2021) + +- upgrading micromamba dependency to 0.8.2 version +- added .robot, .csv, .yaml, .yml, and .json in non-executable fileset +- also added "dot" files as non-executable +- added timestamp update to copyfile functionality +- added toplevel --tag option to allow semantic tagging for client + applications to indicate meaning of rcc execution call + +## v9.7.2 (date: 11.3.2021) + +- adding visibility of installation plans in environment listing +- added --json support to environment listing including installation plan file +- added command `rcc env plan` to show installation plans for environment +- installation plan is now also part of robot diagnostics, if available + +## v9.7.1 (date: 10.3.2021) + +- fixes/improvements to activation and installation plan +- added missing content type to assistant requests +- micromamba upgrade to 0.8.0 + +## v9.7.0 (date: 10.3.2021) + +- conda environments are now activated once on creation, and variables go + with environment, as `rcc_activate.json` +- there is also now new "installation plan" file inside environment, called + `rcc_plan.log` which contains events that lead to activation +- normal runs are now more silent, since details are moved into "plan" file + +## v9.6.2 (date: 5.3.2021) + +- fix for time formats used in timeline, some metrics, and stopwatch + +## v9.6.1 (date: 3.3.2021) + +- refactored code use common.When as consistent timestamp for current rcc run + +## v9.6.0 (date: 3.3.2021) + +- new command `rcc cloud prepare` to support installing assistants on + local computer for faster startup time +- added more timeline entries on relevant parts + +## v9.5.4 (date: 2.3.2021) + +- Updated rpaframework to version 7.6.0 in templates + +## v9.5.3 (date: 2.3.2021) + +- added `--interactive` flag to `rcc task run` command, so that developers + can use debuggers and other interactive tools while debugging + +## v9.5.2 (date: 25.2.2021) + +- bug fix: now cloning sources are not removed during --liveonly action, + even when that source seems to be invalid +- changed timeline to use percent (not permilles anymore) +- minor fix on env diff printout + +## v9.5.1 (date: 25.2.2021) + +- now also printing environment differences when live is dirty and base + is not, just before restoring live from base + +## v9.5.0 (date: 25.2.2021) + +- added support for detecting environment corruption +- now dirhash command can be used to compare environment content + +## v9.4.4 (date: 24.2.2021) + +- fix: added panic protection to telemetry sending, this closes #13 +- added initial support for execution timeline tracking + +## v9.4.3 (date: 23.2.2021) + +- added generic reading and parsing diagnostics for JSON and YAML files + +## v9.4.2 (date: 23.2.2021) + +- fix: marked --report flag required in issue reporting +- added account-email to issue report, as backup contact information + +## v9.4.1 (date: 17.2.2021) + +- added conda.yaml diagnostics (initial take) +- made `rcc env variables` to be not silent anymore +- log level changes in environment creation +- env creation workflow has now 6 steps, added identity visibility + +## v9.4.0 (date: 17.2.2021) + +- added initial robot diagnostics (just robot.yaml for now) +- integrated robot diagnostics into configuration diagnostics (optional) +- integrated robot diagnostics to issue reporting (optional) +- fix: windows paths were wrong; "bin" to "usr" change + +## v9.3.12 (date: 17.2.2021) + +- introduced 48 hour delay to recycling temp folders (since clients depend on + having temp around after rcc process is gone); this closes #12 + +## v9.3.11 (date: 15.2.2021) + +- micromamba upgrade to 0.7.14 +- made process fail early and visibly, if micromamba download fails + +## v9.3.10 (date: 11.2.2021) + +- Windows automation made environments dirty by generating comtypes/gen + folder. Fix is to ignore that folder. +- Added some more diagnostics information. + +## v9.3.9 (date: 8.2.2021) + +- micromamba cleanup bug fix (got error if micromamba is missing) +- micromamba download bug fix (killed on MacOS) + +## v9.3.8 (date: 4.2.2021) + +- making started and finished subprocess PIDs visible in --debug level. + +## v9.3.7 (date: 4.2.2021) + +- micromamba version printout changed, so rcc now parses new format +- micromamba is 0.x, so it does not follow semantic versioning yet, so + rcc will now "lockstep" versions, with micromamba locked to 0.7.12 now + +## v9.3.6 (date: 3.2.2021) + +- removing "defaults" channel from robot templates + +## v9.3.5 (date: 2.2.2021) + +- micromamba upgrade to 0.7.12 +- REGRESSION: `rcc task shell` got broken when micromamba was introduced, + and this version fixes that + +## v9.3.4 (date: 1.2.2021) + +- fix: removing environments now uses rename first and then delete, + to get around windows locked files issue +- warning: on windows, if environment is somehow locked by some process, + this will fail earlier in the process (which is good thing), so be aware +- minor change on cache statistics representation and calculation + +## v9.3.3 (date: 1.2.2021) + +- adding `--dryrun` option to issue reporting + +## v9.3.2 (date: 29.1.2021) + +- added environment variables for installation identity, opt-out status as + `RCC_INSTALLATION_ID` and `RCC_TRACKING_ALLOWED` + +## v9.3.1 (date: 29.1.2021) + +- fix: when environment is leased, temporary folder is will not be recycled +- cleanup command now cleans also temporary folders based on day limit + +## v9.3.0 (date: 28.1.2021) + +- support for applications to submit issue reports thru rcc +- print "robot.yaml" to logs, to make it visible for support cases +- diagnostics can now print into a file, and that is used as part + of issue reporting +- added links to diagnostic checks, for user guidance + +## v9.2.0 (date: 25.1.2021) + +- experiment: carrier PoC + +## v9.1.0 (date: 25.1.2021) + +- new command `rcc configure diagnostics` to help identify environment + related issues +- also requiring new version of micromamba, 0.7.10 + +## v9.0.2 (date: 21.1.2021) + +- fix: prevent direct deletion of leased environment + +## v9.0.1 (date: 20.1.2021) + +- BREAKING CHANGES +- removal of legacy "package.yaml" support + +## v9.0.0 (date: 18.1.2021) + +- BREAKING CHANGES +- new cli option `--lease` to request longer lasting environment (1 hour from + lease request, and next requests refresh the lease) +- new environment variable: `RCC_ENVIRONMENT_HASH` for clients to use +- new command `rcc env unlease` to stop leasing environments +- this breaks contract of pristine environments in cases where one application + has already requested long living lease, and other wants to use environment + with exactly same specification (if pristine, it is shared, otherwise it is + an error) + +## v8.0.12 (date: 18.1.2021) + +- Templates conda -channel ordering reverted pending conda-forge chagnes. + +## v8.0.10 (date: 18.1.2021) + +- fix: when there is no pip dependencies, do not try to run pip command + +## v8.0.9 (date: 15.1.2021) + +- fix: removing one verbosity flag from micromamba invocation + +## v8.0.8 (date: 15.1.2021) + +- now micromamba 0.7.8 is required +- repodata TTL is reduced to 16 hours, and in case of environment creation + failure, fall back to 0 seconds TTL (immediate update) +- using new --retry-with-clean-cache option in micromamba + +## v8.0.7 (date: 11.1.2021) + +- Now rcc manages TEMP and TMP locations for its subprocesses + +## v8.0.6 (date: 8.1.2021) + +- Updated to robot templates +- conda channels in order for `--strict-channel-priority` +- library versions updated and strict as well (rpaframework v7.1.1) +- Added basic guides for what to do in conda.yaml for end-users. + +## v8.0.5 (date: 8.1.2021) + +- added robot test to validate required changes, which are common/version.go + and docs/changelog.md + +## v8.0.4 (date: 8.1.2021) + +- now requires micromamba 0.7.7 at least, with version check added +- micromamba now brings --repodata-ttl, which rcc currently sets for 7 days +- and touching conda caches is gone because of repodata ttl +- can now also cleanup micromamba binary and with --all +- environment validation checks simplified (no more separate space check) + +## v8.0.3 (date: 7.1.2021) + +- adding path validation warnings, since they became problem (with pip) now + that we moved to use micromamba instead of miniconda +- also validation pattern update, with added "~" and "-" as valid characters +- validation is now done on toplevel, so all commands could generate + those warnings (but currently they don't break anything yet) + +## v8.0.2 (date: 5.1.2021) + +- fixing failed robot tests for progress indicators (just tests) + +## v8.0.1 (date: 5.1.2021) + +- added separate pip install phase progress step (just visualization) +- now `rcc env cleanup` has option to remove miniconda3 installation + +## v8.0.0 (date: 5.1.2021) + +- BREAKING CHANGES +- removed miniconda3 download and installing +- removed all conda commands (check, download, and install) +- environment variables `CONDA_EXE` and `CONDA_PYTHON_EXE` are not available + anymore (since we don't have conda installation anymore) +- adding micromamba download, installation, and usage functionality +- dropping 32-bit support from windows and linux, this is breaking change, + so that is why version series goes up to v8 + +## v7.1.5 (date: 4.1.2021) + +- now command `rcc man changelog` shows changelog.md from build moment + +## v7.1.4 (date: 4.1.2021) + +- bug fix for background metrics not send when application ends too fast +- now all telemetry sending happens in background and synchronized at the end +- added this new changelog.md file + +## Older versions + +Versions 7.1.3 and older do not have change log entries. This changelog.md +file was started at 4.1.2021. diff --git a/docs/environment-caching.md b/docs/environment-caching.md new file mode 100644 index 00000000..4ce380fb --- /dev/null +++ b/docs/environment-caching.md @@ -0,0 +1,89 @@ +## What is execution environment isolation and caching? + +Managing and isolating the execution environments is one of the biggest tasks and friction points that RCC is solving. The execution environment needs to have all the pieces in place for the robot to execute and that could mean a full setup Python environment, browsers, custom library packages, etc. In RPA solutions, "Work on my machine" just doesn't cut it. + +![](https://imgs.xkcd.com/comics/python_environment.png) + +*Credit: [xkcd.com](https://xkcd.com/1987/)* + +The list of tools and techniques around just Python environment handling and package management is simply staggering:
+`pyenv, venv, pyenv-virtualenv, pip, pipenv, poetry, conda, ...`.
+Instead, how about just `rcc`? + +RCC creates virtual execution environments that only show up as files and folders on the user machine, basically containing the complexity described above. The target is that neither setting up nor using these environments must either depend on or change anything to the user's environment. + +The [hotel analogy below](/docs/environment-caching.md#a-better-analogy-accommodations) describes the problem and evolution of our solutions. + +## The second evolution of environment management in RCC + +The solution that goes by the working title `Holotree` is a big step up when it comes to efficiency, and it opens up a few significant doors for the future. The name for `Holotree` comes from the analogy to the [Holodeck in Star Trek](https://en.wikipedia.org/wiki/Holodeck). When things changed in the Holodeck, the "reload" only meant that, for example, a chair transformed into a table while the surroundings did not change. + +Holotree is an environment cache that stores each unique file only once. When restoring an execution environment, it moves only the files that need to be moved or removed, reducing the disk usage and file I/O (input/output) actions. Fewer files to move and handle means fewer actions for the virus scanner in the local development setup, and in Robocorp Control Room, reduced CPU time. + +Holotree can fit hundreds of unique environments in a small space. + +![Holotree disk usage](holotree-disk-usage.png) + +*Note: A single execution environment can span from 200MB to >1GB* + +There are other significant changes and improvements in Holotree regarding [relocation]() and [file locking](https://en.wikipedia.org/wiki/File_locking). + +The implementation is already available in [RCC v10](https://github.com/robocorp/rcc/blob/master/docs/changelog.md), and we are rolling out it to our developer tools, applications and Control Room during Q3 2021. + +### Relocation and file locking + +Some binary files have the absolute path written into them when set up, which means the resulting file will only work in the same file structure where it was created. + +To move or [relocate]() the environment, one must either create the same file structure on the target machine or edit the binaries to contain the correct file paths. RCC with Holotree records the environment to the cache (a.k.a Hololib) with the original execution base folder for this problem. Thus, the execution will happen in the same structure but with a mechanism that enables relocation inside that base folder. This avoids file locks and isolates the executions. The mechanism has to do with hashing the execution folder name in the base folder. + +[File locking](https://en.wikipedia.org/wiki/File_locking) affects mainly Windows but can affect macOS and Linux, too. Trying to change a file simultaneously in more than one process is usually harmful and requires individual files for different executors for the execution environments. + +For example, when using Robocorp VS Code extensions on the same machine, it would be nice if the environment files do not get locked. + +For this reason, RCC with Holotree provides a `space` -context so that each execution or client can choose a space where the execution happens. Furthermore, with the partial relocation support described above, the client's space is only made for and used by the clients' discretion, avoiding file collisions and giving control over disk space usage and file I/O to the client applications. + +For example, a normal use case for Robocorp Assistant is that the end-user installs and uses five to ten assistants, and those change pretty rarely. Still, the execution should start pretty much immediately. + +In this case, we would use RCC with Holotree to create a space for each assistant. This way, each assistant has the environment files ready to go with the cost of disk usage. + +On the Workforce side, we could use just a single space as we cannot expect the execution to be similar, but they still share many files. For example, consecutive executions probably have well-over 90% of Python files remain the same, etc. + +## A better analogy: accommodations + +The evolution goes something like this: + +"I invited you to my home." > "Welcome to a hotel built out of ship containers." > "Welcome to an actual Hotel." + +The evolution of these steps is expanded in the following chapters. + +![RCC Environment Hotels](rcc-env-hotels.svg) + +### "I invited you to my home." + +We start with a typical bed-and-breakfast where you set up your own house with kitchen, bathroom, bedroom with fridges, pieces of furniture, etc. Then, depending on the guest, your house is nice and clean after the visit, or the guests trash the place and hide a fish behind the radiator to surprise you a couple of days later. 🐟 + +In the robot dimension, this equates to running pip installs, setting up the correct version of Python and all the dependencies on the machine, executing the robot, and cleaning up all the files and changes made by the execution. A week later, you need to clear the entire pip cache due to some weird collision with another execution. + +### "Welcome to a hotel built out of ship containers." + +As engineers, we try to break a solution down into smaller pieces to mitigate the effects and reduce variables. + +So, after the first guest trashed our house, we change our business model to withstand the occasional rock star or two. 😅 + +We buy shipping containers and deck them out with kitchens, bathrooms, and bedrooms. When we get a visitor, we ask what they want and see if we have a shipping container meeting the requirements or if we need to build a new one. In either case, we get a container for the guest and blob it in our "hotel". Now it does not matter how the guest behaves; after the visit, we can remove the container's content, and the next guest gets a new one. And more importantly, our own house is not damaged. + +In the robot dimension, this is RCC's first environment cache implementation. It uses disk space to provide isolated environments, and if it detects any "pollution" in the environment, it replaces the entire content of the execution folder. The cache always stores the unique pristine version to avoid building from scratch every time. Effective, but not efficient. + +### "Welcome to an actual Hotel." + +After running our "Shipping container Hotel" for a while, we notice we are spending a whole lot of resources to replace the entire container even when the guest only ate a single chocolate bar from the mini-fridge. Also, people asking for different sizes of beds is driving you crazy! + +How about running a regular hotel with some out-of-this-world properties? We still isolate guests to their rooms and protect ourselves from the rock stars, but instead of replacing the entire room every time something is broken or out of place, we send out a janitor and a cleanup crew. + +We also notice that the pristine models of the containers take up a lot of space and have the same furniture. Hence, we decide to store the furniture models as single pieces in one warehouse together with the blueprints of the different room models we have. We also enable the guest to ask for the cleanup crew when making a more extended visit. + +In the robot dimension, this means we have each file once on the Hololib side with the blueprints of the environments we have encountered. + +Each hotel room in the analogy is a Holotree space, and we can load in different content to each space and keep them independent and free from file locking. + +> When we encounter a new environment, we still need to use conda and pip stacks to build the new setup. Environment building will take anything between one to five minutes, depending on the disk and network speeds of the building machine. diff --git a/docs/features.md b/docs/features.md new file mode 100644 index 00000000..2fd63612 --- /dev/null +++ b/docs/features.md @@ -0,0 +1,28 @@ +# Incomplete list of rcc features + +* supported operating systems are Windows, MacOS, and Linux +* support for product families `--robocorp` and `--sema4ai` +* supported sources for environment building are both conda and pypi +* provide repeatable, isolated, and clean environments for automations and + robots to run on +* automatic environment creation based on declarative conda environment.yaml + files +* easily run software robots (automations) based on declarative robot.yaml files +* also support environment creation from package.yaml files +* test robots in isolated environments before uploading them to Control Room +* provide commands for Robocorp runtime and developer tools (Worker, Assistant, + VS Code, ...) +* provides commands to communicate with Robocorp Control Room from command line +* enable caching dormant environments in efficiently and activating them locally + when required without need to reinstall anything +* diagnose robots and network settings to see if there is something that prevents + using robots in specific environment +* support multiple configuration profiles for different network locations and + conditions (remote, office, restricted networks, ...) +* running assistants from command line +* support prebuild environments, where that environment was build elsewhere + and then just imported for local consumption +* allow "mass" prebuilding environments for delivery to those environments + where it is not desired to build those locally +* support unmanaged environments, where rcc only initially build environment + by the spec, but after that, does not do additional management of it diff --git a/docs/history.md b/docs/history.md new file mode 100644 index 00000000..6767441a --- /dev/null +++ b/docs/history.md @@ -0,0 +1,222 @@ +# History of rcc + +This is quick recap of rcc history. Just major topics and breaking changes. +There has already been 500+ commits, with lots of fixes and minor improvements, +and they are not listed here. + +## Version 11.x: between Sep 6, 2021 and ... + +Version "eleven" is work in progress and has already 100+ commits, and at least +following improvements: + +- breaking change: old environment caching (base/live) was fully removed and + holotree is only solution available +- breaking change: hashing algorithm changed, holotree uses siphash fron now on +- environment section of commands were removed, replacements live in holotree + section +- environment cleanup changed, since holotree is different from base/live envs +- auto-scaling worker count is now based on number of CPUs minus one, but at + least two and maximum of 96 +- templates can now be automatically updated from Cloud and can also be + customized using settings.yaml autoupdates section +- added option to do strict environment building, which turns pip warnings + into actual errors +- added support for speed test, where current machine performance gets scored +- hololib.zip files can now be imported into normal holotree library (allows + air gapped workflow) +- added more commands around holotree implementation +- added support for preRunScripts, which are executed in similar context that + actual robot will use, and there can be OS specific scripts only run on + that specific OS +- added profile support with define, export, import, and switch functionality +- certificate bundle, micromambarc, piprc, and settings can be part of profile +- `settings.yaml` now has layers, so that partial settings are possible, and + undefined ones use internal default settings +- `docs/` folder has generated "table of content" +- introduced "shared holotree", where multiple users in same computer can + share resources needed by holotree spaces +- in addition to normal tasks, now robot.yaml can also contain devTasks, which + can be activated with flag `--dev` +- holotrees can also be imported directly from URLs +- some experimental support for virtual environments (pyvenv.cfg and others) +- moved from "go-bindata" to use new go buildin "embed" module +- holotree now also fully support symbolic links inside created environments +- improved cleanup in relation to new shared holotrees +- individual catalog removal and cleanup is now possible +- prebuild environments can now be forced using "no build" configurations + +## Version 10.x: between Jun 15, 2021 and Sep 1, 2021 + +Version "ten" had 32 commits, and had following improvements: + +- breaking change: removed lease support +- listing of dependencies is now part of holotree space (golden-ee.yaml) +- dependency listing is visible before run (to help debugging environment + changes) and there is also command to list them +- environment definitions can now be "freezed" using freeze file from run output +- supporting multiple environment configurations to enable operating system + and architecture specific freeze files (within one robot project) +- made environment creation serialization visible when multiple processes are + involved +- added holotree check command to verify holotree library integrity and remove + those items that are broken + +## Version 9.x: between Jan 15, 2021 and Jun 10, 2021 + +Version "nine" had 101 commits, and had following improvements: + +- breaking change: old "package.yaml" support was fully dropped +- breaking change: new lease option breaks contract of pristine environments in + cases where one application has already requested long living lease, and + other wants to use environment with exactly same specification +- new environment leasing options added +- added configuration diagnostics support to identify environment related issues +- diagnostics can also be done to robots, so that robot issues become visible +- experiment: carrier robots as standalone executables +- issue reporting support for applications (with dryrun options) +- removing environments now uses rename/delete pattern (for detecting locking + issues) +- environment based temporary folder management improvements +- added support for detecting when environment gets corrupted and showing + differences compared to pristine environment +- added support for execution timeline summary +- assistants environments can be prepared before they are used/needed, and this + means faster startup time for assistants +- environments are activated once, on creation (stored on `rcc_activate.json`) +- installation plan is also stored as `rcc_plan.log` inside environment and + there is command to show it +- introduction of `settings.yaml` file for configurable items +- introduced holotree command subtree into source code base +- holotree implementation is build parallel to existing environment management +- holotree now co-exists with old implementation in backward compatible way +- exporting holotrees as hololib.zip files is possible and robot can be executed + against it +- micromamba download is now done "on demand" only +- result of environment variables command are now directly executable +- execution can now be profiled "on demand" using command line flags +- download index is generated directly from changelog content +- started to use capability set with Cloud authorization +- new environment variable `ROBOCORP_OVERRIDE_SYSTEM_REQUIREMENTS` to make + skip those system requirements that some users are willing to try +- new environment variable `RCC_VERBOSE_ENVIRONMENT_BUILDING` to make + environment building more verbose +- for `task run` and `task testrun` there is now possibility to give additional + arguments from commandline, by using `--` separator between normal rcc + arguments and those intended for executed robot +- added event journaling support, and command to see them +- added support to run scripts inside task environments + +## Version 8.x: between Jan 4, 2021 and Jan 18, 2021 + +Version "eight" had 14 commits, and had following improvements: + +- breaking change: 32-bit support was dropped +- automatic download and installation of micromamba +- fully migrated to micromamba and removed miniconda3 +- no more conda commands and also removed some conda variables +- now conda and pip installation steps are clearly separated + +## Version 7.x: between Dec 1, 2020 and Jan 4, 2021 + +Version "seven" had 17 commits, and had following improvements: + +- breaking change: switched to use sha256 as hashing algorithm +- changelogs are now held in separate file +- changelogs are embedded inside rcc binary +- started to introduce micromamba into project +- indentity.yaml is saved inside environment +- longpath checking and fixing for Windows introduced +- better cleanup support for items inside `ROBOCORP_HOME` + +## Version 6.x: between Nov 16, 2020 and Nov 30, 2020 + +Version "six" had 24 commits, and had following improvements: + +- breaking change: stdout is used for machine readable output, and all error + messages go to stderr including debug and trace outputs +- introduced postInstallScripts into conda.yaml +- interactive create for creating robots from templates + +## Version 5.x: between Nov 4, 2020 and Nov 16, 2020 + +Version "five" had 28 commits, and had following improvements: + +- breaking change: REST API server removed (since it is easier to use just as + CLI command from applications) +- Open Source repository for rcc created and work continued there (Nov 10) +- using Apache license as OSS license +- detecting interactive use and coloring outputs +- tutorial added as command +- added community pull and tooling support + +## Version 4.x: between Oct 20, 2020 and Nov 2, 2020 + +Version "four" had 12 commits, and had following improvements: + +- breaking change related to new assistant encryption scheme +- usability improvements on CLI use +- introduced "controller" concept as toplevel persistent option +- dynamic ephemeral account support introduced + +## Version 3.x: between Oct 15, 2020 and Oct 19, 2020 + +Version "three" had just 6 commits, and had following improvements: + +- breaking change was transition from "task" to "robotTaskName" in robot.yaml +- assistant heartbeat introduced +- lockless option introduced and better support for debugging locking support + +## Version 2.x: between Sep 16, 2020 and Oct 14, 2020 + +Version "two" had around 29 commits, and had following improvements: + +- URL (breaking) changes in Cloud required Major version upgrade +- added assistant support (list, run, download, upload artifacts) +- added support to execute "anything", no condaConfigFile required +- file locking introduced +- robot cache introduced at `$ROBOCORP_HOME/robots/` + +## Version 1.x: between Sep 3, 2020 and Sep 16, 2020 + +Version "one" had around 13 commits, and had following improvements: + +- terminology was changed, so code also needed to be changed +- package.yaml converted to robot.yaml +- packages were renamed to robots +- activities were renamed to tasks +- added support for environment cleanups +- added support for library management + +## Version 0.x: between April 1, 2020 and Sep 8, 2020 + +Even when project started as "conman", it was renamed to "rcc" on May 8, 2020. + +Initial "zero" version was around 120 commits and following highlevel things +were developed in that time: + +- cross-compiling to Mac, Linux, Windows, and Raspberry Pi +- originally supported were 32 and 64 bit architectures of arm and amd +- delivery as signed/notarized binaries in Mac and Windows +- download and install miniconda3 automatically +- management of separate environments +- using miniconda to manage packages at `ROBOCORP_HOME` +- merge support for multiple conda.yaml files +- initially using miniconda3 to create those environments +- where robots were initially defined in `package.yaml` +- packaging and unpacking of robots to and from zipped activity packages +- running robots (using run and testrun subcommands) +- local conda channels and pip wheels +- sending metrics to cloud +- CLI handling and command hierarchy using Viper and Cobra +- cloud communication using accounts, credentials, and tokens +- `ROBOCORP_HOME` variable as center of universe +- there was server support, and REST API for applications to use +- ignore files support +- support for embedded templates using go-bindata +- originally used locality-sensitive hashing for conda.yaml identity +- both Lab and Worker support + +## Birth of "Codename: Conman" + +First commit to private conman repo was done on April 1, 2020. And name was +shortening of "conda manager". And it was developer generated name. diff --git a/docs/holotree-disk-usage.png b/docs/holotree-disk-usage.png new file mode 100644 index 00000000..0e95e846 Binary files /dev/null and b/docs/holotree-disk-usage.png differ diff --git a/docs/maintenance.md b/docs/maintenance.md new file mode 100644 index 00000000..4233001b --- /dev/null +++ b/docs/maintenance.md @@ -0,0 +1,89 @@ +# Holotree and library maintenance + +This documentation section gives you brief ideas how to maintain your +holotree/hololib setup. + +## Why do maintenance? + +There are number of reasons for doing maintenance, some of which are: + +- running into problem with using holotree, and wanting to start over +- something breaks in holotree/lib and there is need to fix it +- running out of disk space, and wanting to reduced used foot print +- remove old/unused spaces from holotree +- remove old/unused catalogs from hololib +- just to keep things running smoothly on future robot invocations + +## Shared holotree and maintenance + +When doing maintenance in shared holotree, you should be aware, that it might +affect other user accounts in same machine. So when you are doing system wide +maintenance in shared holotree, make sure that nothing is working on those +environments and catalogs that your maintenance targets to. + +## Maintenance vs. tools using holotrees + +When doing maintenance on any holotree, you should be aware, that if Robocorp +tooling (Worker, Assistant, VS Code plugins, rcc, ...) is +also at same time using same holotree/hololib, your maintenance actions might +have negative effect on those tools. + +Some of those effects might be: + +- wiping environment under tool using it and causing automation, debugging, + editing, or development tooling to crash or produce unexpected results +- if catalog or space was removed, and it is needed later, then that must + be rebuild or downloaded, and that will slow down that use-case +- removing catalogs or hololib entries that will be needed by automations + or tooling, might cause slowness when needed next time, or if builds are + prevented it might even deny usage of those spaces + +## Maintenace and product families + +Since v18 of rcc, there are two different product families present. To +explicitely maintain specific product family holotree, then either +`--robocorp` or `--sema4ai` flag should be given. Both product families +have their separate holotree libraries and spaces. + +## Deleting catalogs and spaces + +Before you delete anything, you should be aware of those things and what is +there. + +Catalogs can be listed using `rcc holotree catalogs` command, and +if you add `--identity` you can see what was their environment specification. + +Then command `rcc holotree list` is used to list those concrete spaces that +are consuming your disk space. There you can also see how many times space +has been used, and when was last time it was used. (And using in this context +means, that rcc did create or refresh that specific space.) + +Once you know what is there, and there are needs to remove catalogs, then +see `rcc holotree remove -h` for more about information on that. One good +option to use there is `--check 5` to also cleanup all released spare parts. + +And to free disk space consumed by concrete holotrees, see command +`rcc holotree delete -h`, which can be used to delete those spaces that +are not needed anymore. + +## Keeping hololib consistent + +And in cases, where there are holotree restoration problems, or hololib +issues, it is good to run consistency checks against that hololib. This +can be done using `rcc holotree check -h` command. And good option there +is to add `--retries 5` option, to get more "garbage collection cycles" +to maintain used disk space. + +Note that after running this command, and if there was something broken +inside hololib, then some of your catalogs have been removed, and in this +case it is good thing, since they were broken. And if they are needed in +future, those should be either build or imported. + +## Summary of maintenance related commands + +- `rcc holotree list -h` lists holotree spaces and their location +- `rcc holotree catalogs -h` list known catalogs, their blueprints, and stats +- `rcc configuration cleanup -h` for general cleanup procedures +- `rcc holotree delete -h` for deleting individual spaces +- `rcc holotree remove -h` for removing individual catalogs +- `rcc holotree check -h` for checking integrity of hololib diff --git a/docs/profile_configuration.md b/docs/profile_configuration.md new file mode 100644 index 00000000..0937b060 --- /dev/null +++ b/docs/profile_configuration.md @@ -0,0 +1,68 @@ +# Profile Configuration + +## What is profile? + +Profile is way to capture configuration information related to specific +network location. System can have multiple profiles defined, but only one +can be active at any moment. + +### When do you need profiles? + +- if you are in restricted network where direct network access is not available +- if you are working in multiple locations with different access policies + (for example switching between office, hotel, airport, or remote locations) +- if you want to share your working setup with others in same network + +### What does it contain? + +- information from `settings.yaml` (can be partial) +- configuration for micromamba ("micromambarc" is almost like "condarc") +- configuration for pip (pip.ini or piprc) +- root certificate bundle in pem format +- proxy settings (`HTTP_PROXY` and `HTTPS_PROXY`) +- options for `ssl-verify` and `ssl-no-revoke` + +## Quick start guide + +### Setup Utility -- user interface for this + +More behind [this link](https://robocorp.com/docs/control-room/setup-utility). + +### Pure rcc workflow for handling existing profiles + +```sh +# import that Office profile, so that it can be used +rcc configuration import --filename profile_office.yaml + +# start using that Office profile +rcc configuration switch --profile Office + +# verify that basic things work by doing diagnostics +rcc configuration diagnostics + +# when basics work, see if full environment creation works +rcc configuration speedtest + +# when you want to reset profile to "system default" state +# in practice this means that all settings files removed +rcc configuration switch --noprofile + +# if you want to export profile and deliver to others +rcc configuration export --profile Office --filename shared.yaml +``` + +## What is needed? + +- you need rcc 11.9.10 or later +- your existing `settings.yaml` file (optional) +- your existing `micromambarc` file (optional) +- your existing `pip.ini` file (optional) +- your existing `cabundle.pem` file (optional) +- knowledge about your network proxies and certificate policies + +## Discovery process + +1. You must be inside that network that you are targetting the configuration. +2. Run Setup Utility and use it to setup and verify your profile. +3. Export profile and share it with rest of your team/organization. +4. Create other profiles for different network locations (remote, VPN, ...) diff --git a/docs/rcc-env-hotels.svg b/docs/rcc-env-hotels.svg new file mode 100644 index 00000000..01fdbd15 --- /dev/null +++ b/docs/rcc-env-hotels.svg @@ -0,0 +1,390 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/recipes.md b/docs/recipes.md new file mode 100644 index 00000000..f63b47b2 --- /dev/null +++ b/docs/recipes.md @@ -0,0 +1,1003 @@ +# Tips, tricks, and recipies + + +## How to see dependency changes? + +Since version 10.2.2, rcc can show dependency listings using +`rcc robot dependencies` command. Listing always have two sided, "Wanted" +which is content from dependencies.yaml file, and "Available" which is from +actual environment command was run against. Listing is also shown during +robot runs. + +### Why is this important? + +- as time passes and world moves forward, new version of used components + (dependencies) are released, and this may cause "configuration drift" on + your robots, and without tooling in place, this drift might go unnoticed +- if your dependencies are not fixed, there will be configuration drift and + your robot may change behaviour (become buggy) when dependency changes and + goes against implemented robot +- even if you fix your dependencies in `conda.yaml`, some of those components + or their components might have floating dependencies and they change your + robots behaviour +- if your execution environment is different from your development environment + then there might be different versions available for different operating + systems +- if dependency resolution algorithm changes (pip for example) then you might + get different environment with same `conda.yaml` +- when you upgrade one of your dependencies (for example, rpaframework) to new + version, dependency resolution will now change, and now listing helps you + understand what has changed and how you need to change your robot + implementation because of that + +### Example of dependencies listing from holotree environment + +```sh +# first list dependencies from execution environment +rcc robot dependencies --space user + +# if everything looks good, export it as wanted dependencies.yaml +rcc robot dependencies --space user --export + +# and verify that everything looks `Same` +rcc robot dependencies --space user +``` + + +## How to freeze dependencies? + +Starting from rcc 10.3.2, there is now possibility to freeze dependencies. +This is how you can experiment with it. + +### Steps + +- have your `conda.yaml` to contain only those dependencies that your robot + needs, either with exact versions or floating ones +- run robot in your target environment at least once, so that environment + there gets created +- from that run's artifact directory, you should find file that has name + something like `environment_xxx_yyy_freeze.yaml` +- copy that file back into your robot, right beside existing `conda.yaml` + file (but do not overwrite it, you need that later) +- edit your `robot.yaml` file at `condaConfigFile` entry, and add your + newly copied `environment_xxx_yyy_freeze.yaml` file there if it does not + already exist there +- repackage your robot and now your environment should stay quite frozen + +### Limitations + +- this is new and experimental feature, and we don't know yet how well it + works in all cases (but we love to get feedback) +- currently this freezing limits where robot can be run, since dependencies + on different operating systems and architectures differ and freezing cannot + be done in OS and architecture neutral way +- your robot will break, if some specific package is removed from pypi or + conda repositories +- your robot might also break, if someone updates package (and it's dependencies) + without changing its version number +- for better visibility on configuration drift, you should also have + `dependencies.yaml` inside your robot (see other recipe for it) + + +## How pass arguments to robot from CLI? + +Since version 9.15.0, rcc supports passing arguments from CLI to underlying +robot. For that, you need to have task in `robot.yaml` that co-operates with +additional arguments appended at the end of given `shell` command. + +### Example robot.yaml with scripting task + +```yaml +tasks: + Run all tasks: + shell: python -m robot --report NONE --outputdir output --logtitle "Task log" tasks.robot + + scripting: + shell: python -m robot --report NONE --outputdir output --logtitle "Scripting log" + +condaConfigFile: conda.yaml +artifactsDir: output +PATH: + - . +PYTHONPATH: + - . +ignoreFiles: + - .gitignore +``` + +### Run it with `--` separator. + +```sh +rcc task run --interactive --task scripting -- --loglevel TRACE --variable answer:42 tasks.robot +``` + + +## How to run any command inside robot environment? + +Since version 9.20.0, rcc now supports running any command inside robot space +using `rcc task script` command. + +### Some example commands + +Run following commands in same direcotry where your `robot.yaml` is. Or +otherwise you have to provide `--robot path/to/robot.yaml` in commandline. + +```sh +# what python version we are running +rcc task script --silent -- python --version + +# get pip list from this environment +rcc task script --silent -- pip list + +# start interactive ipython session +rcc task script --interactive -- ipython +``` + + +## How to convert existing python project to rcc? + +### Basic workflow to get it up and running + +1. Create a new robot using `rcc create` with `Basic Python template`. +2. Remove task.py and and copy files from your existing project to this new + rcc/robot project. +3. Discover all your publicly available dependencies (including your python + version) and try find as many as possible from https://anaconda.org/conda-forge/ + and take rest from https://pypi.org/ and put those dependencies + into `conda.yaml`. And remove all those dependencies that you do not actually + need in your project. +4. Do not add any private dependencies into `conda.yaml`, and also no passwords + in that `conda.yaml` either (passwords belong to secure place, like Vault). +5. Modify your `robot.yaml` task definitions so, that it is how your python + project should be executed. +6. If you have additional private libraries, put them inside robot directory + structure (like under `libraries` or something similar) and edit PYTHONPATH + settings in `robot.yaml` to include those paths (relative paths only). +7. If you have additional scripts/small binaries that your robot dependes on, + add them inside robot directory structure (like under `scripts` directory) + and edit PATH settings in `robot.yaml` to include that (relative) path. +8. If your python project needs external dependencies (like Word or Excel) + then those dependencies must be present in machine where robot is executed + and they are not part of this conversion. +9. Run robot and test if it works, and iterate to make needed changes. + +### What next? + +* Your python project is now converted to rcc and should be locally "runnable". +* Setup Assistant or Worker in your machine and create Assistant or Robot + in Robocorp Control Room, and try to run it from there. +* If your robot is "headless", has all dependencies, and should be runnable + in Linux, then you can try to run it in container from Control Room. +* If your project is python2 project, then consider converting it to python3. +* If you want to use `rpaframework` in your robot (like dialogs for example), + then you have to start converting to use those features in your code. +* etc. + + +## Is rcc limited to Python and Robot Framework? + +Absolutely not! Here is something completely different for you to think about. + +Lets assume, that you are in almost empty Linux machine, and you have to +quickly build new micromamba in that machine. Hey, there is `bash`, `$EDITOR`, +and `curl` here. But there are no compilers, git, or even python installed. + +> Pop quiz, hot shot! Who you gonna call? MacGyver! + +### This is what we are going to do ... + +Here is set of commands we are going to execute in our trusty shell + +```sh +mkdir -p builder/bin +cd builder +$EDITOR robot.yaml +$EDITOR conda.yaml +$EDITOR bin/builder.sh +curl -o rcc https://downloads.robocorp.com/rcc/releases/v10.3.2/linux64/rcc +chmod 755 rcc +./rcc run -s MacGyver +``` + +### Write a robot.yaml + +So, for this to be a robot, we need to write heart of our robot, which is +`robot.yaml` of course. + +```yaml +tasks: + Âĩmamba: + shell: builder.sh +condaConfigFile: conda.yaml +artifactsDir: output +PATH: +- bin +``` + +### Write a conda.yaml + +Next, we need to define what our robot needs to be able to do our mighty task. +This goes into `conda.yaml` file. + +```yaml +channels: +- conda-forge +dependencies: +- git +- gmock +- cli11 +- cmake +- compilers +- cxx-compiler +- pybind11 +- libsolv +- libarchive +- libcurl +- gtest +- nlohmann_json +- cpp-filesystem +- yaml-cpp +- reproc-cpp +- python=3.8 +- pip=20.1 +``` + +### Write a bin/builder.sh + +And finally, what does our robot do. And this time, this goes to our directory +bin, which is on our PATH, and name for this "robot" is actually `builder.sh` +and it is a bash script. + +```sh +#!/bin/bash -ex + +rm -rf target output/micromamba* +git clone https://github.com/mamba-org/mamba.git target +pushd target +version=$(git tag -l --sort='-creatordate' | head -1) +git checkout $version +mkdir -p build +pushd build +cmake .. -DCMAKE_INSTALL_PREFIX=/tmp/mamba -DENABLE_TESTS=ON -DBUILD_EXE=ON -DBUILD_BINDINGS=OFF +make +popd +popd +mkdir -p output +cp target/build/micromamba output/micromamba-$version +``` + + +## Think what you can do with this conda.yaml? + +``` +channels: + # Just using conda-forge, nothing else. + - conda-forge + +dependencies: + # I'm not going to have python directly installed here .. + # But let's go wild with conda-forge ... + + - nginx=1.21.6 # https://anaconda.org/conda-forge/nginx + - php=8.1.5 # https://anaconda.org/conda-forge/php + - go=1.17.8 # https://anaconda.org/conda-forge/go + - postgresql=14.2 # https://anaconda.org/conda-forge/postgresql + - terraform=1.1.9 # https://anaconda.org/conda-forge/terraform + - awscli=1.23.9 # https://anaconda.org/conda-forge/awscli + - firefox=100.0 # https://anaconda.org/conda-forge/firefox +``` + +## How to control holotree environments? + +There is three controlling factors for where holotree spaces are created. + +First is location of `ROBOCORP_HOME` at creation time of environment. This +decides general location for environment and it cannot be changed or relocated +afterwards. + +Second controlling factor is given using `--controller` option and default for +this is value `user`. And when applications are calling rcc, they should +have their own "controller" identity, so that all spaces created for one +application are groupped together by prefix of their "space" identity name. + +Third controlling factor is content of `--space` option and again default +value there is `user`. Here it is up to user or application to decide their +strategy of use of different names to separate environments to their logical +used partitions. If you choose to use just defaults (user/user) then there +is going to be only one real environment available. + +But above three controls gives you good ways to control how you and your +applications manage their usage of different python environments for +different purposes. You can share environments if you want, but you can also +give a dedicated space for those things that need full control of their space. + +So running following commands demonstrate different levels of control for +space creation. + +``` +export ROBOCORP_HOME=/tmp/rchome +rcc holotree variables simple.yaml +rcc holotree variables --space tips simple.yaml +rcc holotree variables --controller tricks --space tips simple.yaml +``` + +If you now run `rcc holotree list` it should list something like following. + +``` +Identity Controller Space Blueprint Full path +-------- ---------- ----- -------- --------- +5a1fac3c5_2daaa295 rcc.user tips c34ed96c2d8a459a /tmp/rchome/holotree/5a1fac3c5_2daaa295 +5a1fac3c5_9fcd2534 rcc.user user c34ed96c2d8a459a /tmp/rchome/holotree/5a1fac3c5_9fcd2534 +9e7018022_2daaa295 rcc.tricks tips c34ed96c2d8a459a /tmp/rchome/holotree/9e7018022_2daaa295 +``` + +### How to get understanding on holotree? + +See: https://github.com/robocorp/rcc/blob/master/docs/environment-caching.md + +### How to activate holotree environment? + +On Linux/MacOSX: + +```sh +# full robot environment +source <(rcc holotree variables --space mine --robot path/to/robot.yaml) + +# or with just conda.yaml +source <(rcc holotree variables --space mine path/to/conda.yaml) +``` + +On Windows + +```sh +rcc holotree variables --space mine --robot path/to/robot.yaml > mine_activate.bat +call mine_activate.bat +``` + +You can also try + +```sh +rcc task shell --robot path/to/robot.yaml +``` + + +## What is `ROBOCORP_HOME`? + +It is environment variable level settings, that says where Robocorp tooling +can keep tooling specific files and configurations. It has default values, +and normal case is that defaults are fine. But if there is need to "relocate" +that somewhere else, then this environment variable does the trick. + +### Are there some rules for `ROBOCORP_HOME` variable? + +- go with defaults, unless you have very good reason to override it +- avoid using spaces or special characters in path that is `ROBOCORP_HOME`, + so stick to basic english letters and numbers +- never use your "home" directory as `ROBOCORP_HOME`, it will cause conflicts +- never share `ROBOCORP_HOME` between two users, it should be unique to each + different user account +- also keep it private and protected, other users should not have access + to that directory +- never use `ROBOCORP_HOME` as working directory for user, or any other + tools; this directory is only meant for Robocorp tooling to use, change, + and operate on +- never put `ROBOCORP_HOME` on network drive, since those tend to be slow, + and using those can cause real performance issues +- always make sure, that user owning that `ROBOCORP_HOME` directory has full + control access and permissions to everything inside that directory structure + + +### When you might actually need to setup `ROBOCORP_HOME`? + +- if your username contains spaces, or some special characters that can cause + tooling to break +- if path to your home directory is very long, it might cause long path issues, + and one way to go around is have `ROBOCORP_HOME` on shorter path +- if you need to have `ROBOCORP_HOME` on some different disk than default +- if your home directory is on HDD drive (or even network drive), but you + have fast SSD direve available, performance might be much better on SSD + +## What is shared holotree? + +Shared holotree is way to multiple users use same environment blueprint in +same machine, or even in different machines with same, once it is built or +imported into hololib. + +## How to setup rcc to use shared holotree? + +### One time setup + +On each machine, where you want to use shared holotree, the shared location +needs to be enabled once. +This depends on the operating system so the commands below are OS specific +and do require elevated rights from the user that runs them. + +The commands to enable the shared locations are: +* Windows: `rcc holotree shared --enable` + * Shared location: `C:\ProgramData\robocorp` +* MacOS: `sudo rcc holotree shared --enable` + * Shared location: `/Users/Shared/robocorp` +* Linux: `sudo rcc holotree shared --enable` + * Shared location: `/opt/robocorp` + +Note: On Windows the command below assumes the standard `BUILTIN\Users` +user group is present. +If your organization has replaced this you can grant the permission with: + +``` +icacls "C:\ProgramData\robocorp" /grant "*S-1-5-32-545:(OI)(CI)M" /T +``` + +To switch the user to using shared holotrees use the following command. + +```sh +rcc holotree init +``` + +### Reverting back to private holotrees + +If user wants to go back to private holotrees, they can run following command. + +```sh +rcc holotree init --revoke +``` + +## What can be controlled using environment variables? + +- `ROBOCORP_HOME` points to directory where rcc keeps most of Robocorp related + files and directories are kept +- `ROBOCORP_OVERRIDE_SYSTEM_REQUIREMENTS` makes rcc more relaxed on system + requirements (like long path support requirement on Windows) but it also + means that if set, responsibility of resolving failures are on user side +- `RCC_VERBOSE_ENVIRONMENT_BUILDING` makes environment creation more verbose, + so that failing environment creation can be seen with more details +- `RCC_CREDENTIALS_ID` is way to provide Control Room credentials using + environment variables +- `RCC_NO_BUILD` with any non-empty value will prevent rcc for creating + new environments (also available as `--no-build` CLI flag, and as + an option in `settings.yaml` file) +- `RCC_VERBOSITY` controls how verbose rcc output will be. If this variable + is not set, then verbosity is taken from `--silent`, `--debug`, and `--trace` + CLI flags. Valid values for this variable are `silent`, `debug` and `trace`. +- `RCC_NO_TEMP_MANAGEMENT` with any non-empty value will prevent rcc for + doing any management in relation to temporary directories; using this + environment variable means, that something else is managing temporary + directories life cycles (and this might also break environment isolation) +- `RCC_NO_PYC_MANAGEMENT` with any non-empty value will prevent rcc for + doing any .pyc file management; using this environment variable means, that + something else is doing that management (and using this makes rcc slower + and hololibs become bigger and grow faster, since .pyc files are unfriendly + to caching) + + +## How to troubleshoot rcc setup and robots? + +```sh +# to get generic setup diagnostics +rcc configure diagnostics + +# to get robot and environment setup diagnostics +rcc configure diagnostics --robot path/to/robot.yaml + +# to see how well rcc performs in your machine +rcc configure speedtest +``` + +### Additional debugging options + +- generic flag `--debug` shows debug messages during execution +- generic flag `--trace` shows more verbose debugging messages during execution +- flag `--timeline` can be used to see execution timeline and where time was spent +- with option `--pprof ` enable profiling if performance is problem, + and want to help improve it (by submitting that profile file to developers) + +## Advanced network diagnostics + +When using custom endpoints or just needing more control over what network +checks are done, command `rcc configure netdiagnostics` may become helpful. + +```sh +# to test advanced network diagnostics with defaults +rcc configure netdiagnostics + +# to capture advanced network diagnostics defaults to new configuration file +rcc configure netdiagnostics --show > path/to/modified.yaml + +# to test advanced network diagnostics with custom tests +rcc configure netdiagnostics --checks path/to/modified.yaml +``` + +### Configuration + +- get example configuration out using `--show` option (as seen above) +- configuration file format is YAML +- add or remove points to DNS, HTTP HEAD and GET methods +- `url:` and `codes:` are required fields for HEAD and GET checks +- `codes:` field is list of acceptable HTTP response codes +- `content-sha256` is optional, and provides additional confidence when content + is static and result content hash can be calculated (using sha256 algorithm) + +## What is in `robot.yaml`? + +### Example + +```yaml +tasks: + Just a task: + robotTaskName: Just a task + Version command: + shell: python -m robot --version + Multiline command: + command: + - python + - -m + - robot + - --report + - NONE + - -d + - output + - --logtitle + - Task log + - tasks.robot + +devTasks: + Editor setup: + shell: python scripts/editor_setup.py + Repository update: + shell: python scripts/repository_update.py + +condaConfigFile: conda.yaml + +environmentConfigs: +- environment_linux_amd64_freeze.yaml +- environment_windows_amd64_freeze.yaml +- common_linux_amd64.yaml +- common_windows_amd64.yaml +- common_linux.yaml +- common_windows.yaml +- conda.yaml + +preRunScripts: +- privatePipInstall.sh +- initializeKeystore.sh + +artifactsDir: output + +ignoreFiles: +- .gitignore + +PATH: +- . +- bin + +PYTHONPATH: +- . +- libraries +``` + +### What is this `robot.yaml` thing? + +It is declarative description in [YAML format](https://en.wikipedia.org/wiki/YAML) +of what robot is and what it can do. + +It is also a pointer to "a robot center of a universe" for directory it resides. +So it is marker of "current working folder" when robot starts to execute and +that will be indicated in `ROBOT_ROOT` environment variable. All declarations +inside `robot.yaml` should be relative to and inside of this location, so do +not use absolute paths here, or relative references to any parent directory. + +It also marks root location that gets wrapped into `robot.zip` when either +wrapping locally or pushing to Control Room. Nothing above directory holding +`robot.yaml` gets wrapped into that zip file. + +Also note that `robot.yaml` is just a name of a file. Other names can be used +and then given to commands using `--robot othername.yaml` CLI option. But +in Robocorp tooling, this default name `robot.yaml` is used to have common +ground without additional configuration needs. + +### Why "the center of the universe"? + +Firstly, it is not "the center", it is just "a center of a universe" for +specific robot. So it only applies to that specific robot, when operations +are done around that one specific robot. Other robots have their own centers. + +And reason for thinking this way is, that it is "convention over configuration", +meaning that when we have this concept, there is much less configuration to do. +It gives following things automatically, without additional configuration: + +- what is "root" folder, when wrapping robot into deliverable package +- what is starting working directory when robot is executed (robot itself can + of course change its working directory freely while running) +- it gives solid starting point for relative paths inside robot, so that + PATH, PYTHONPATH, artifactsDir, and other relative references can be + converted absolute ones +- it allows robot location to be different for different users and on different + machines, and still have everything declared with known (but relative) + locations + +### What are `tasks:`? + +One robot can do multiple tasks. Each task is a single declaration of named +task that robot can do. + +There are three types of task declarations: + +1. The `robotTaskName` form, which is simplest and there only name of a task + is given. In above example `Just a task` is a such thing. This is Robot + Framework specific form. +2. The `shell` form, where full CLI command is given as oneliner. In above + example, `Version command` is example of this. +3. The `command` form is oldest. It is given as list of command and its + arguments, and it is most accurate way to declare CLI form, but it is also + most spacious form. + +### What are `devTasks:`? + +They are tasks like above `tasks:` define. But they have two major differences +compared to normal `tasks:` definitions: + +1. They are for developers at development machines, for doing development time + activities and tasks. They should never be available in cloud containers, + Assistants or Agents. Developer tools can provide support for them, but + their semantics should be only valid in development context. +2. They can be run like normal tasks, by providing `--dev` flag. But during + their run, all `preRunScripts:` are ignored. Otherwise environment is + created and managed as with normal tasks, but without pre-run scripts + applied. + +The `devTasks:` primary goal is to provide developers a way to use the same +tooling to automate their development process as normal `tasks:` provide ways +to automate robot actions. Some examples could be: common editor setups, +version control repository updates. + +Currently `--dev` option is only available for `rcc run` and `rcc task run` +commands. With the `--dev` option the only available tasks for execution will +be the `devTasks:`. The normal `tasks:` will be skipped/missing. If the `--dev` +option is missing, the `devTasks:` will be skipped/missing, and the normal +`tasks:` will be the ones available for execution. + +### What is `condaConfigFile:`? + +> Use of this is deprecated, please use `environmentConfigs:` instead. + +This is actual name used as `conda.yaml` environment configuration file. +See next topic about details of `conda.yaml` file. +This is just single file that describes dependencies for all operating systems. +For more versatile selection, see `environmentConfigs` below. If that +`environmentConfigs` exists and one of those files matches machine running +rcc, then this config is ignored. + +### What are `environmentConfigs:`? + +These are like condaConfigFile above, but as priority list form. First matching +and existing item from that list is used as environment configuration file. + +These files are matched by operating system (windows/darwin/linux) and by +architecture (amd64/arm64). If filename contains word "freeze", it must +match OS and architecture exactly. Other variations allow just some or none +of those parts. + +And if there is no such file, then those entries are just ignored. And if +none of files match or exist, then as final resort, `condaConfigFile` value +is used if present. + +### What are `preRunScripts:`? + +This is set of scripts or commands that are run before actual robot task +execution. Idea with these scripts is that they can be used to customize +runtime environment right after it has been restored from hololib, and just +before actual robot execution is done. + +If script names contains some of "amd64", "arm64", "darwin", "windows" and/or +"linux" words (like `script_for_amd64_linux.sh`) then other architectures +and operating systems will skip those scripts, and only amd64 linux systems +will execute them. + +All these scripts are run in "robot" context with all same environment +variables available as in robot run. + +These scripts can pollute the environment, and it is ok. Next rcc operation +on same holotree space will first do the cleanup though. + +All scripts must be executed successfully or otherwise full robot run is +considered failure and not even tried. Scripts should use exit code zero +to indicate success and everything else is failure. + +Some ideas for scripts could be: + +- install custom packages from private pip repository +- use Vault secrets to prepare system for actual robot run +- setup and customize used tools with secret or other private details that + should not be visible inside hololib catalogs (public caches etc) + +### What is `artifactsDir:`? + +This is location of technical artifacts, like log and freezefiles, that are +created during robot execution and which can be used to find out technical +details about run afterwards. Do not confuse these with work-item data, which +are more business related and do not belong here. + +During robot run, this locations is available using `ROBOT_ARTIFACTS` +environment variable, if you want to store some additional artifacts there. + +### What are `ignoreFiles:`? + +This is a list of configuration files that rcc uses as locations for ignore +patterns used while wrapping robot into a robot.zip file. But note, that once +filename is on this list, it must also be present on directory structure, this +is part of a contract. + +Content of those files should be similar to what is used normally as version +control systems as ignore files (like .gitignore file in git context). +Here rcc implements only subset of functionality, and allows just mostly +globbing patterns or exact names of files and directories. + +Note: do not put file or directory names that you want to be ignored directly +in this list. They all should reside in one of those configurations listed in +this configuration list. + +Tip: using `.gitignore` as one of those `ignoreFiles:` entries helps you to +remove duplication of maintenance pressures. But if you want ignore different +things in git and in robot.zip, or if there are conflicts between those, +feel free use different filenames as you see fit. + +### What are `PATH:`? + +This allows adding entries into `PATH` environment variable. Intention +is to allow something like `bin` directory inside robot, where custom +scripts and binaries can be located and available for execution during +robot run. + +### What are `PYTHONPATH:`? + +This allows adding entries into `PYTHONPATH` environment variable. Intention +is to allow something like `libraries` directory inside robot, where custom +libraries can be located and automatically loaded by python and robot. + + +## What is in `conda.yaml`? + +### Example + +```yaml +channels: +- conda-forge + +dependencies: +- python=3.9.13 +- nodejs=16.14.2 +- pip=22.1.2 +- pip: + - robotframework-browser==12.3.0 + - rpaframework==15.6.0 + +rccPostInstall: + - rfbrowser init +``` + +### What is this `conda.yaml` thing? + +It is declarative description in [YAML format](https://en.wikipedia.org/wiki/YAML) +of environment that should be set up. + +### What are `channels:`? + +Channels are conda sources where to get packages to be used in setting up +environment. It is recommended to use `conda-forge` channel, but there are +others also. Other recommendation is that only one channel is used, to get +consistently build environments. + +Channels should be in priority order, where first one has highest priority. + +Example above uses `conda-forge` as its only channel. +For more details about conda-forge, see this [link.](https://anaconda.org/conda-forge) + +### What are `dependencies:`? + +These are libraries that are needed to be installed in environment that is +declared in this `conda.yaml` file. By default they come from locations +setup in `channels:` part of file. + +But there is also `- pip:` part and those dependenies come from +[PyPI](https://pypi.org/) and they are installed after dependencies from +`channels:` have been installed. + +In above example, `python=3.9.13` comes from `conda-forge` channel. +And `rpaframework==15.6.0` comes from [PyPI](https://pypi.org/project/rpaframework/). + +### What are `rccPostInstall:` scripts? + +Once environment dependencies have been installed, but before it is frozen as +hololib catalog, there is option to run some additional commands to customize +that environment. It is list of "shell" commands that are executed in order, +and if any of those fail, environment creation will fail. + +All those scripts must come from package declared in `dependencies:` section, +and should not use any "local" knowledge outside of environment under +construction. This makes environment creation repeatable and cacheable. + +Do not use any private or sensitive information in those post install scripts, +since result of environment build could be cached and visible to everybody +who has access to that cache. If you need to have private or sensitive packages +in your environment, see `preRunScripts` in `robot.yaml` file. + + +## How to do "old-school" CI/CD pipeline integration with rcc? + +If you have CI/CD pipeline and want to updated your robots from there, this +recipe should give you ideas how to do it. This example works in linux, and +you probably have to modify it to work on Mac or Windows, but idea will be same. + +Basic requirements are: +- have well formed robot in version control +- have rcc command available or possibility to fetch it +- possibility on CI/CD pipeline to run just simple CLI commands + +### The oldschoolci.sh script + +```sh +#!/bin/sh -ex + +curl -o rcc https://downloads.robocorp.com/rcc/releases/v11.14.3/linux64/rcc +chmod 755 rcc +./rcc cloud push --account ${ACCOUNT_ID} --directory ${ROBOT_DIR} --workspace ${WORKSPACE_ID} --robot ${ROBOT_ID} +``` + +So above script uses `curl` command to download rcc from download site, and +makes it executable. And then it simply calls that `rcc` command, and expects +that CI system has provided few variables. + +### A setup.sh script for simulating variable injection. + +```sh +#!/bin/sh + +export ACCOUNT_ID=4242:cafe9d9c0dadag00d37b9577babe1575b67bc1bbad3ce9484dead36a649c865beef26297e67c8d94f0f0057f0100ab64:https://api.eu1.robocorp.com +export WORKSPACE_ID=1717 +export ROBOT_ID=2121 +export ROBOT_DIR=$(pwd)/therobot +``` + +Expectations for above setup are: +- robot to be updated is in EU1 (behind https://api.eu1.robocorp.com API) +- Control Room account has "Access creadentials" 4242 available and active +- account has access to workspace 1717 +- there exist previously created robot 2121 in that workspace +- robot is located in "therobot" directory directly under "current working + directory" (centered around `robot.yaml` file) +- and account has suitable rights to actually push robot to Control Room + +### Simulating actual CI/CD step in local machine. + +```sh +#!/bin/sh -ex + +source setup.sh +./oldschoolci.sh +``` + +Above script brings "setup" and "old school CI" together, but just for +demonstration purposes. For real life use, adapt and remember security (no +compromising variable content inside repository). + +### Additional notes + +- if CI/CD worker/container can be custom build, then it is recommended to + download rcc just once and not on every run (like oldschoolci.sh script now + does) +- that `ACCOUNT_ID` should be stored in credentials store/vault in CI system, + because that is secret that you need to use to be able to push to cloud +- that `ACCOUNT_ID` is "ephemeral" account, and will not be saved in `rcc.yaml` +- also consider saving other variables in secure way +- in actual CI/CD pipeline, you might want to embed actual commands into + CI step recipe and not have external scripts (but you decide that) + + +## How to setup custom templates? + +Custom templates allows making your own templates that can be used when +new robot is created. So if you have your own standard way of doing things, +then custom template is good way to codify it. + +You then need to do these steps: + +- setup custom settings.yaml that point location where template configuration + file is located (the templates.yaml file) +- if you are using profiles, then make above change in settings.yaml used there +- create that custom templates.yaml configuration file that lists available + templates, and where template bundle can be found (the templates.zip file) +- and finally build that templates.zip to bundle together all those templates + that were listed in configuration file +- and finally both templates.yaml and templates.zip must be somewhere behind + URL that starts with https: + +Note: templates are needed only on development context, and they are not used +or needed in Assistant or Worker context. + +### Custom template configuration in `settings.yaml`. + +In settings.yaml, there is `autoupdates:` section, and there is entry for +`templates:` where you should put exact name and location where active +templates configuration file is located. + +Example: + +```yaml +autoupdates: + templates: https://special.acme.com/robot/templates-1.0.1.yaml +``` + +As above example shows, name is configurable, and can even contain some +versioning information, if so needed. + +### Custom template configuration file as `templates.yaml`. + +In that `templates.yaml` following things must be provided: + +- `hash:` (sha256) of "templates.zip" file (so that integrity of templates.zip + can be verified) +- `url:` to exact name and location where that templates.zip can be downloaded +- `date:` when this template.yaml file was last updated +- `templates:` as key/value pairs of templates and their "one liner" + description seen in UIs +- so, if there is `shell.zip` inside templates.zip, then that should have + `shell: Shell Robot Template` or something similar in that `templates:` + section + +Example: + +```yaml +hash: c7b1ba0863d9f7559de599e3811e31ddd7bdb72ce862d1a033f5396c92c5c4ec +url: https://special.acme.com/robot/templates-1.0.1.zip +date: 2022-09-12 +templates: + shell: Simple Shell Robot template + extended: Extended Robot Framework template + playwright: Playwright template + producer-consumer: Producer-consumer model template +``` + +### Custom template content in `templates.zip` file. + +Then that `templates.zip` is zip-of-zips. So for each key from templates.yaml +`templates:` sections should have matching .zip file inside that master zip. + +### Shared using `https:` protocol ... + +Then both `templates.yaml` and `templates.zip` should be hosted somewhere +which can be accessed using https protocol. Names there should match those +defined in above steps. + +And that `settings.yaml` should either be delivered standalone into those +developer machines that need to use those templates, or better yet, be part +of "profile" that developers can use to setup all of required configurations. + + +## Where can I find updates for rcc? + +https://downloads.robocorp.com/rcc/releases/index.html + +That is rcc download site with two categories of: +- tested versions (these are ones we ship with our tools) +- latest 20 versions (which are not battle tested yet, but are bleeding edge) + + +## What has changed on rcc? + +### See changelog from git repo ... + +https://github.com/robocorp/rcc/blob/master/docs/changelog.md + +### See that from your version of rcc directly ... + +```sh +rcc docs changelog +``` + + +## Can I see these tips as web page? + +Sure. See following URL. + +https://github.com/robocorp/rcc/blob/master/docs/recipes.md + diff --git a/docs/robocorp_stack.png b/docs/robocorp_stack.png index 41299e84..9d5fac44 100644 Binary files a/docs/robocorp_stack.png and b/docs/robocorp_stack.png differ diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md new file mode 100644 index 00000000..a3829bd6 --- /dev/null +++ b/docs/troubleshooting.md @@ -0,0 +1,111 @@ +# Troubleshooting guidelines and known solutions + +> Help us to help you to resolve issue you are having. + +## Tools to help with troubleshooting issues + +- run command `rcc configuration diagnostics` and see if there are warnings, + failures or errors in output (and same with `rcc configuration netdiagnostics`) +- run command `rcc configuration tlsprobe` against various host:port targets + to get insights about supported TLS versions, and certificates used there +- if failure is with specific robot, then try running command + `rcc configuration diagnostics --robot path/to/robot.yaml` and see if + those robot diagnostics have something that identifies a problem (or to get + only robot diagnostics, you can also use `rcc robot diagnostics` command) +- run command `rcc configuration speedtest` to see, if problem is actually + performance related (like slow disk or network access) +- run command `rcc holotree check --retries 5` to verify (and fix possibly) + problems inside hololib storage +- run rcc commands with `--debug` and `--timeline` flags, and see if anything + there adds more information on why failure is happening + +## How to troubleshoot issue you are having? + +- are you using latest versions of tools and libraries, and if not, then first + thing to do is update them and then retry +- if you have only encoutered the problem just once, try to repeat it, and + if you cannot repeat it, you are done +- has this ever before worked in this same user/machine/network combination, + and if not, then are you using correct profile and settings? +- if this worked previously, and then stopped working, what has changed or + what have you changed? any updates? new IT policies? new network location? +- gather evidence, that is all logs, console outputs, stack traces, screenshots, + and look them thru +- what is first error your see in console output, and what is last error your + see, then look between +- also look thru all warnings and failures too + +## Reporting an issue + +- describe what were you trying to achieve +- describe what did you actually do when trying to achieve it +- describe what did actually happen +- describe what were you expecting to happen +- describe what did happen that you think indicates that there is an issue +- describe what error messages did you see +- describe steps that are needed to be able to reproduce this issue +- describe what have you already tried to resolve this issue +- describe what has changed since this issue was not present and when everything + worked ok +- you should share your `conda.yaml` used with robot or environment +- you should share your `robot.yaml` that defines your robot +- you should share your code, or minimal sample code, that can reproduce + problem you are having +- provide all evidence that you have gathered (as native form as possible, + and as fully as possible; do not truncate stack traces or logs unnecessarily) + +## Network access related troubleshooting questions + +- are you behind proxy, firewall, VPN, endpoint security solutions, or any + combination of those? +- if you are, do you know, what brand are those products, and are they + provided by third party service providers, and who are those third parties? +- are all those services configured to allow access to essential network places + so that they don't cause interference on cloud access (change request or + response headers, filter out URL parameters, change request or response + bodies, disallow DNS resolution, etc.)? +- if those services require additional configuration in robot running machine, + are those configurations in place in profiles used by rcc (service URLs, + usernames and passwords, custom certificates, etc.)? +- if profile is in place, is that specific user account switched to use that + profile? +- are there errors or warnings on `rcc configuration diagnostics` or in + `rcc configuration netdiagnostics` runs? +- does `rcc configuration speedtest` work, and how does performance look like? + +## Known solutions + +### Access denied while building holotree environment (Windows) + +If file is .dll or .exe file, then there is probably some process running, that +has actually locked that file, and tooling cannot complete its operation while +that other process is running. Other process might be virus scanner, some other +tool (Assistant, Worker, Automation Studio, VS Code, rcc) using same +environment, or even open Explorer view. + +To resolve this, close other applications, or wait them to finish before trying +same operation again. + +### Message "Serialized environment creation" repeats + +There can be few reasons for this. Here are some ways to resolve it. + +If multiple robots in same machine are trying to create new environment or +refresh existing one at exactly same time, then only one of them can continue. +This is there to protect integrity and security of holotree and hololib, and +also conserve resources for doing duplicate work. In this case, best thing to +resolve this is just to wait processes to complete. + +Other case is where there are multiple rcc processes running, but none of them +seems to be progressing. This might be indication that there is one "zombie" +process, which is holding on to a lock, and wont go away since some of its +child processes is still running (like python, web browser, or Excel). In this +case, best way is to close those "hanging" processes, and let OS to finish +that pending (and lock holding) process. + +Third case is where there seems to be only one rcc, and it is just waiting and +repeating that message. In this case it is probably a permission issue, and +for some reason .lck file is not accessible/lockable by rcc process. In this +case, you should go and look if current user has rights to actually modify +those .lck files, and if not, you have to grant them those. This might require +administrator privileges to actually change those file permissions. diff --git a/docs/usecases.md b/docs/usecases.md new file mode 100644 index 00000000..35baa7c1 --- /dev/null +++ b/docs/usecases.md @@ -0,0 +1,37 @@ +# Incomplete list of rcc use cases + +* run robots in Robocorp Worker locally or in cloud containers +* run robots in Robocorp Assistant +* provide commands for Robocorp Code to develop robots locally and + communicate to Robocorp Control Room +* provide commands that can be used in CI pipelines (Jenkins, Gitlab CI, ...) + to push robots into Robocorp Control Room +* can also be used to run robot tests in CI/CD environments +* provide isolated environments to run python scripts and applications +* to use other scripting languages and tools available from conda-forge (or + conda in general) with isolated and easily installed manner (see list below + for ideas what is available) +* provide above things in computers, where internet access is restricted or + prohibited (using pre-made hololib.zip environments, or importing prebuild + environments build elsewhere) +* pull and run community created robots without Control Room requirement +* use rcc provided holotree environments as soft-containers (they are isolated + environments, but also have access to rest of your machine resources) + +## What is available from conda-forge? + +* python and libraries +* ruby and libraries +* perl and libraries +* lua and libraries +* r and libraries +* julia and libraries +* make, cmake and compilers (C++, Fortran, ...) +* nodejs +* nginx +* rust +* php +* go +* gawk, sed, and emacs, vim +* ROS libraries (robot operating system) +* firefox diff --git a/docs/venv.md b/docs/venv.md new file mode 100644 index 00000000..5983484f --- /dev/null +++ b/docs/venv.md @@ -0,0 +1,95 @@ +# Support for virtual environments + +There is now experimental feature in rcc to create virtual environments on top +of rcc holotree environment. + +## What does it do? + +When you run command `rcc venv`, that will do following things: +- using given `conda.yaml` file, it will create that base environment +- this new environment will be "unmanaged" holotree space, so once created, rcc + wont touch base environment (unless forced) +- it will also be "externally managed" PEP-668 environment, so no additional + things should be pip installed in that environment +- then on top of that base environment, rcc will try automatically create local + project specific `venv` directory and list available activation commands +- in addition to that, rcc also puts experimental `depxtraction.py` script in + same directory (more on that later in this document) + +## How to get started? + +- first you need rcc version 17.17.0 or later available in your system +- then on some directory, you should have `conda.yaml` for base environment, + something like this: + +``` +channels: +- conda-forge +dependencies: +- python=3.10.12 +- pip=23.2.1 +- robocorp-truststore=0.8.0 +``` + +- then in that directory, run command `rcc venv conda.yaml` +- after that, you should see list of activation commands to use this new venv +- after activation, you can use normal pip commands to populate that venv as + you wish + +## Limitations of `rcc venv`: + +- currently naming and location is fixed, so you cannot change those +- this venv is always build on top of holotree space, so that holotree space + must always be there +- and that space is "unmanaged", so idea is, that once created, it is developer + responsibility to delete or force update it if dependencies change +- also other things installed from conda-forget from underlying holotree space + are hidden and only python environment is visible + +## Dangers of using `--force` in `rcc venv` context. + +- unmanged holotrees are not user specific, so be careful when using `--force` + option to recreate those spaces, and recommendation is to use `--space` and + `--controller` options to limit usage to your intentions +- be aware that `--force` makes three things to happen +- first it is needed if `venv` was already created (so rcc wont overwrite + things in venv, unless you really force it) +- second it is used to tell rcc, that also underlying holotree space should + be recreated (maybe with conflicting dependencies) +- third, it forces also full holotree space installation and updating caches + +## What is this `depxtraction.py` thing? + +It is "dependency extraction", with limitations (see below). + +Idea behing `depxtraction.py` is, that when there is modified environment, +where additional dependencies are installed using tools like pip or poetry, +those dependencies can be extracted by tooling into simple `conda.yaml` +format. + +## Limitations of `depxtraction.py`: + +- no conda dependencies detected, and every dependency that python tooling + reports are expected to be from PyPI (except "hardcoded" python, pip and + robocorp-truststore that are defined as bootstrapping dependencies from + conda-forge) +- if there are deeply recursive dependencies (X depends on Y depends on Z + depends on X) then currently those dependencies will vanish, since "root" + dependency is unclear (if you run these cases, please report those, so + that better functionality can be implemented and tested) +- only top level dependencies are resolved and versioned, and listed as + dependencies; subdependency resolving is left for pip resolver to figure out +- and because individual install commands can create inconsistent environment, + it is possible, that once `conda.yaml` is generated out of such environment, + recreation of such environment might actually fail to resolve correctly and + in those cases, you have to adjust generated `conda.yaml` accordingly + +## Ideas for usage + +- start VS Code from CLI inside activated environment +- create rcc venv, install packages manually inside that environment, + make your automation work, once automation is working so far, extract + dependencies, recreate rcc venv and continue iterating ... +- try to run `depxtraction.py` on your system python setup and see what + comes up there (this can be done by just using `depxtraction.py` with + your system python without activating any virtual environments) diff --git a/docs/vocabulary.md b/docs/vocabulary.md new file mode 100644 index 00000000..72b7ed7c --- /dev/null +++ b/docs/vocabulary.md @@ -0,0 +1,137 @@ +# Vocabulary + +## Blueprint + +Is unique identity calculated from `conda.yaml` or in general from some +`environment.yaml` file after it is formatted in canonical, unified form. +Currently `rcc` uses "siphash" for that. This is form of a fingerprint. + +## Catalog + +Catalog is description how final created environment should look like, after +it is expanded and relocated into target location. Catalog is metadata, and +is used to verify that created environment matches original specification. + +## Controller + +This is tool or context that is currently running `rcc` command. + +## Diagnostics + +Network, configuration, or robot diagnostics, that are executed to give +status of one or some of those aspects. + +## Dirty environment + +An environment, holotree space specially, become dirty when after it is +restored in pristine state, something adds, deletes, or modifies files or +directories inside that specific space. This can happen by preRunScripts +modifying something when robot start, robot itself doing something that +changes actual environment, or when someone intentionally tries to modify +or install something into environment manually. + +When dirtyness is desired thing, like for developer purposes, use unmanaged +holotree spaces. But for normal automations and robots, it is good to start +from pristine, clean state. + +## Environment + +An environment here means either concrete holotree space, which contains +set of code and libraries (like python runtime environment) that are needed +for running specific robot or automation. Or it means that same environment +but as stashed away building blocks that are stored in hololib. + +## Fingerprint + +Fingerprint is normally a hash digest calculated from some content. Various +algorithms can be used for this, and some examples are Sha256 and siphash. + +## Holotree + +Is set of working areas, where concrete robots can run. Robots and processes +run inside one of these instances. These consume disk space. These are also +resetted into pristine state each time one of `rcc` run or environment related +subcommands are executes. + +## Hololib + +Is set of building blocks that are used to setup concreate holotree spaces. +Hololib contains both library and catalogs. Every unique content is stored +only once in library part. And catalogs refers to library fingerprints to +identify what parts they use. + +## Identity + +Identity is something that describes or identifies something uniquely. +For example `identity.yaml` is description that equals to `conda.yaml`. + +## Platform + +Platform refers to either Windows, MacOS, or Linux. And also either "amd64" +or "arm64" architectures of those. + +## Prebuild environment + +Prebuild environment is something that contains building blocks for full +holotree space in form of catalog + hololib parts. It is per operating system +and architecture, and can only be used in shared holotree context, where +parts are relocatable between different user accounts. + +During building concrete holotree space from prebuild environment, there is +no need for internet connection. If something inside robot run needs internet, +then that is not prebuild environment concern. + +## Pristine environment + +Environment that is restored to match exactly original, specified state. +When environments are used and content inside is changed, then those are +dirty/corrupted environments. They can be restored back to pristine state +using `rcc` commands. + +## Private holotree + +This is state, where all environments are created for single user, and cannot +be shared between users. These must be build and managed privately and using +them normally requires internet access. + +## Product family + +This is reference to either Robocorp products or Sema4.ai products. + +## Profile + +Profile is set of settings that describes network and Robocorp configurations +so that cloud and Control Room can be used in robot context. + +## Robot + +Robot is automation or process, that will be running inside one of concrete +holotree space. + +## Shared holotree + +This is state, where created environment can be relocated and different users +can use same shared catalogs to quickly replicate environments with identical +specifications, but provided for each user as separate space. + +## Space + +Concrete created environment where processes and robot actually run. Each +holotree space is identified by three things: user, controller, and space +identifier. Each different combination of those values receives their own +separate directory. These will each separately consume diskspace. + +## Unmanaged holotree space + +This is holotree space, that is created by `rcc` but it is not managed by +`rcc` after it gets created. It is up to user or using tool to manage and +maintain that environment. It can get dirty, can have traditional tooling +adding dependencies there, and it can deviate from specification. + +Note: unmanaged holotree spaces are not user specific, and managing access +to those spaces is left to tooling/users who use these unmanaged spaces. + +## User + +User account identity that is using `rcc`. Users wont share concrete holotrees +in shared holotree context. Each user will get their own separate space. diff --git a/fail/fail_test.go b/fail/fail_test.go new file mode 100644 index 00000000..45b9226f --- /dev/null +++ b/fail/fail_test.go @@ -0,0 +1 @@ +package fail_test diff --git a/fail/handling.go b/fail/handling.go new file mode 100644 index 00000000..f57a554a --- /dev/null +++ b/fail/handling.go @@ -0,0 +1,44 @@ +package fail + +import ( + "errors" + "fmt" +) + +func Around(err *error) { + original := recover() + if original == nil { + return + } + + catch, ok := original.(delimited) + if !ok { + panic(original) + } + + *err = catch() +} + +func Fast(err error) { + if err != nil { + panic(failure("%v", err)) + } +} + +func On(condition bool, form string, details ...interface{}) { + if condition { + panic(failure(form, details...)) + } +} + +func failure(form string, details ...interface{}) delimited { + err := errors.New(form) + if len(details) > 0 { + err = fmt.Errorf(form, details...) + } + return func() error { + return err + } +} + +type delimited func() error diff --git a/go.mod b/go.mod index 9c9ef3a4..438cd34b 100644 --- a/go.mod +++ b/go.mod @@ -1,27 +1,42 @@ module github.com/robocorp/rcc -go 1.14 +go 1.20 require ( - github.com/fsnotify/fsnotify v1.4.9 - github.com/glaslos/tlsh v0.2.1-0.20190803090415-ef1954596284 - github.com/go-bindata/go-bindata v3.1.2+incompatible // indirect + github.com/dchest/siphash v1.2.3 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 - github.com/gorilla/mux v1.7.4 - github.com/manifoldco/promptui v0.8.0 - github.com/mattn/go-isatty v0.0.12 - github.com/mitchellh/go-homedir v1.1.0 - github.com/mitchellh/mapstructure v1.2.2 // indirect - github.com/pelletier/go-toml v1.6.0 // indirect - github.com/spf13/afero v1.2.2 // indirect - github.com/spf13/cast v1.3.1 // indirect - github.com/spf13/cobra v0.0.7 - github.com/spf13/jwalterweatherman v1.1.0 // indirect + github.com/mattn/go-isatty v0.0.17 + github.com/mitchellh/go-ps v1.0.0 + github.com/spf13/cobra v1.7.0 + github.com/spf13/viper v1.17.0 + golang.org/x/sys v0.13.0 + golang.org/x/term v0.13.0 + gopkg.in/yaml.v2 v2.4.0 +) + +exclude ( + golang.org/x/crypto v0.0.0 + golang.org/x/crypto v0.13.0 +) + +require ( + github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/magiconair/properties v1.8.7 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/pelletier/go-toml/v2 v2.1.0 // indirect + github.com/sagikazarmark/locafero v0.3.0 // indirect + github.com/sagikazarmark/slog-shim v0.1.0 // indirect + github.com/sourcegraph/conc v0.3.0 // indirect + github.com/spf13/afero v1.10.0 // indirect + github.com/spf13/cast v1.5.1 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/spf13/viper v1.6.2 - golang.org/x/sys v0.0.0-20200331124033-c3d80250170d // indirect - golang.org/x/text v0.3.2 // indirect - golang.org/x/tools v0.0.0-20200331202046-9d5940d49312 // indirect - gopkg.in/ini.v1 v1.55.0 // indirect - gopkg.in/yaml.v2 v2.2.8 + github.com/subosito/gotenv v1.6.0 // indirect + go.uber.org/atomic v1.9.0 // indirect + go.uber.org/multierr v1.9.0 // indirect + golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect + golang.org/x/text v0.13.0 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 25df6784..63384bfa 100644 --- a/go.sum +++ b/go.sum @@ -1,222 +1,510 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= -github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/glaslos/tlsh v0.2.0 h1:9zr1gNyYCAMMsirzU5FFlUEEWp5hsrFE+B4LZEg8psk= -github.com/glaslos/tlsh v0.2.0/go.mod h1:S/OBGINihiGogV6WoaLeMY2UrS5Rl1iqMnplLonIOI4= -github.com/glaslos/tlsh v0.2.1-0.20190803090415-ef1954596284 h1:AWxbfoKclxMucKNN1csTKj1OqypRsbjnBtVpfMUzeuQ= -github.com/glaslos/tlsh v0.2.1-0.20190803090415-ef1954596284/go.mod h1:Fg7YBN7EUtifZmdJrQOQHvebtw5RF89IX7nWFsmaqeE= -github.com/go-bindata/go-bindata v3.1.2+incompatible h1:5vjJMVhowQdPzjE1LdxyFF7YFTXg5IgGVW4gBr5IbvE= -github.com/go-bindata/go-bindata v3.1.2+incompatible/go.mod h1:xK8Dsgwmeed+BBsSy2XTopBn/8uK2HWuGSnA11C3Joo= -github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/dchest/siphash v1.2.3 h1:QXwFc8cFOR2dSa/gE6o/HokBMWtLUaNDVd+22aKHeEA= +github.com/dchest/siphash v1.2.3/go.mod h1:0NvQU092bT0ipiFN++/rXm69QG9tVxLAlQHIXMPAkHc= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc= -github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= -github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/juju/ansiterm v0.0.0-20180109212912-720a0952cc2a h1:FaWFmfWdAUKbSCtOU2QjDaorUexogfaMgbipgYATUMU= -github.com/juju/ansiterm v0.0.0-20180109212912-720a0952cc2a/go.mod h1:UJSiEoRfvx3hP73CvoARgeLjaIOjybY9vj8PUPPFGeU= -github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/lunixbochs/vtclean v0.0.0-20180621232353-2d01aacdc34a h1:weJVJJRzAJBFRlAiJQROKQs8oC9vOxvm4rZmBBk0ONw= -github.com/lunixbochs/vtclean v0.0.0-20180621232353-2d01aacdc34a/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= -github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY= -github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= -github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/manifoldco/promptui v0.8.0 h1:R95mMF+McvXZQ7j1g8ucVZE1gLP3Sv6j9vlF9kyRqQo= -github.com/manifoldco/promptui v0.8.0/go.mod h1:n4zTdgP0vr0S3w7/O/g98U+e0gwLScEXGwov2nIKuGQ= -github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4= -github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.2.2 h1:dxe5oCinTXiTIcfgmZecdCzPmAJKd46KsCWc35r0TV4= -github.com/mitchellh/mapstructure v1.2.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml v1.6.0 h1:aetoXYr0Tv7xRU/V4B4IZJ2QcbtMUFoNb3ORp7TzIK4= -github.com/pelletier/go-toml v1.6.0/go.mod h1:5N711Q9dKgbdkxHL+MEfF31hpT7l0S0s/t2kKREewys= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= +github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mitchellh/go-ps v1.0.0 h1:i6ampVEEF4wQFF+bkYfwYgY+F/uYJDktmvLPf7qIgjc= +github.com/mitchellh/go-ps v1.0.0/go.mod h1:J4lOc8z8yJs6vUwklHw2XEIiT4z4C40KtWVN3nvg8Pg= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= +github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= -github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc= -github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= -github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.7 h1:FfTH+vuMXOas8jmfb5/M7dzEYx7LpcLb7a0LPe34uOU= -github.com/spf13/cobra v0.0.7/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= -github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= -github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= -github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sagikazarmark/locafero v0.3.0 h1:zT7VEGWC2DTflmccN/5T1etyKvxSxpHsjb9cJvm4SvQ= +github.com/sagikazarmark/locafero v0.3.0/go.mod h1:w+v7UsPNFwzF1cHuOajOOzoq4U7v/ig1mpRjqV+Bu1U= +github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= +github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= +github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= +github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= +github.com/spf13/afero v1.10.0 h1:EaGW2JJh15aKOejeuJ+wpFSHnbd7GE6Wvp3TsNhb6LY= +github.com/spf13/afero v1.10.0/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ= +github.com/spf13/cast v1.5.1 h1:R+kOtfhWQE6TVQzY+4D7wJLBgkdVasCEFxSUBYBYIlA= +github.com/spf13/cast v1.5.1/go.mod h1:b9PdjNptOpzXr7Rq1q9gJML/2cdGQAo69NKzQ10KN48= +github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= +github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.4.0 h1:yXHLWeravcrgGyFSyCgdYpXQ9dR9c/WED3pg1RhxqEU= -github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= -github.com/spf13/viper v1.6.2 h1:7aKfF+e8/k68gda3LOjo5RxiUqddoFxVq4BKBPrxk5E= -github.com/spf13/viper v1.6.2/go.mod h1:t3iDnF5Jlj76alVNuyFBk5oUMCvsrkbvZK0WQdfDi5k= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= -github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/spf13/viper v1.17.0 h1:I5txKw7MJasPL/BrfkbA0Jyo/oELqVmux4pR/UxOMfI= +github.com/spf13/viper v1.17.0/go.mod h1:BmMMMLQXSbcHK6KAOiFLz0l5JHrU89OdIRHvsk0+yVI= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= +go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= +go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200331124033-c3d80250170d h1:nc5K6ox/4lTFbMVSL9WRR81ixkcwXThoiF6yf+R9scA= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200331202046-9d5940d49312 h1:2PHG+Ia3gK1K2kjxZnSylizb//eyaMG8gDFbOG7wLV8= -golang.org/x/tools v0.0.0-20200331202046-9d5940d49312/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= -gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.55.0 h1:E8yzL5unfpW3M6fz/eB7Cb5MQAYSZ7GKo4Qth+N2sgQ= -gopkg.in/ini.v1 v1.55.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= -gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/hamlet/hamlet.go b/hamlet/hamlet.go index 7cdba2f6..94c649bb 100644 --- a/hamlet/hamlet.go +++ b/hamlet/hamlet.go @@ -29,7 +29,6 @@ when used in code. I like my specifications short and declarative, not longwindy and procedural code form. One line, one expectation! - */ package hamlet diff --git a/htfs/commands.go b/htfs/commands.go new file mode 100644 index 00000000..a42be109 --- /dev/null +++ b/htfs/commands.go @@ -0,0 +1,281 @@ +package htfs + +import ( + "fmt" + "os" + "os/user" + "path/filepath" + "runtime" + "strings" + + "github.com/robocorp/rcc/anywork" + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/conda" + "github.com/robocorp/rcc/fail" + "github.com/robocorp/rcc/journal" + "github.com/robocorp/rcc/pathlib" + "github.com/robocorp/rcc/pretty" + "github.com/robocorp/rcc/robot" + "github.com/robocorp/rcc/settings" + "github.com/robocorp/rcc/xviper" +) + +type CatalogPuller func(string, string, bool) error + +func NewEnvironment(condafile, holozip string, restore, force bool, puller CatalogPuller) (label string, scorecard common.Scorecard, err error) { + defer fail.Around(&err) + + who, _ := user.Current() + host, _ := os.Hostname() + pretty.Progress(0, "Context: %q <%v@%v> [%v/%v].", who.Name, who.Username, host, common.Platform(), settings.OperatingSystem()) + + if common.WarrantyVoided() { + tree, err := New() + fail.On(err != nil, "%s", err) + + path := tree.WarrantyVoidedDir([]byte(common.ControllerIdentity()), []byte(common.HolotreeSpace)) + return path, common.NewScorecard(), err + } + + journal.CurrentBuildEvent().StartNow(force) + + if settings.Global.NoBuild() { + pretty.Note("'no-build' setting is active. Only cached, prebuild, or imported environments are allowed!") + } + + haszip := len(holozip) > 0 + if haszip { + common.Debug("New zipped environment from %q!", holozip) + } + + path, externally := "", "" + defer func() { + if err != nil { + pretty.Regression(15, "Holotree restoration failure, see above [with %d workers on %d CPUs].", anywork.Scale(), runtime.NumCPU()) + } else { + pretty.Progress(15, "Fresh %sholotree done [with %d workers on %d CPUs].", externally, anywork.Scale(), runtime.NumCPU()) + if len(path) > 0 { + usefile := fmt.Sprintf("%s.use", path) + pathlib.AppendFile(usefile, []byte{'.'}) + } + } + if haszip { + pretty.Note("There is hololib.zip present at: %q", holozip) + } + if len(path) > 0 { + dependencies := conda.LoadWantedDependencies(conda.GoldenMasterFilename(path)) + dependencies.WarnVulnerability( + "https://robocorp.com/docs/faq/openssl-cve-2022-11-01", + "HIGH", + "openssl", + "3.0.0", "3.0.1", "3.0.2", "3.0.3", "3.0.4", "3.0.5", "3.0.6") + } + }() + if common.SharedHolotree { + pretty.Progress(1, "Fresh [shared mode] holotree environment %v. (parent/pid: %d/%d)", xviper.TrackingIdentity(), os.Getppid(), os.Getpid()) + } else { + pretty.Progress(1, "Fresh [private mode] holotree environment %v. (parent/pid: %d/%d)", xviper.TrackingIdentity(), os.Getppid(), os.Getpid()) + } + + lockfile := common.HolotreeLock() + completed := pathlib.LockWaitMessage(lockfile, "Serialized environment creation [holotree lock]") + locker, err := pathlib.Locker(lockfile, 30000, common.SharedHolotree) + completed() + fail.On(err != nil, "Could not get lock for holotree. Quiting.") + defer locker.Release() + + _, holotreeBlueprint, err := ComposeFinalBlueprint([]string{condafile}, "") + fail.Fast(err) + + common.EnvironmentHash, common.FreshlyBuildEnvironment = common.BlueprintHash(holotreeBlueprint), false + pretty.Progress(2, "Holotree blueprint is %q [%s with %d workers on %d CPUs from %q].", common.EnvironmentHash, common.Platform(), anywork.Scale(), runtime.NumCPU(), filepath.Base(condafile)) + journal.CurrentBuildEvent().Blueprint(common.EnvironmentHash) + + tree, err := New() + fail.Fast(err) + + if !haszip && !tree.HasBlueprint(holotreeBlueprint) && common.Liveonly { + tree = Virtual() + common.Timeline("downgraded to virtual holotree library") + } + if common.UnmanagedSpace { + tree = Unmanaged(tree) + } + fail.Fast(tree.ValidateBlueprint(holotreeBlueprint)) + scorecard = common.NewScorecard() + var library Library + if haszip { + library, err = ZipLibrary(holozip) + fail.On(err != nil, "Failed to load %q -> %s", holozip, err) + common.Timeline("downgraded to holotree zip library") + } else { + scorecard.Start() + fail.Fast(RecordEnvironment(tree, holotreeBlueprint, force, scorecard, puller)) + library = tree + } + + if restore { + pretty.Progress(14, "Restore space from library [with %d workers on %d CPUs; with compression: %v].", anywork.Scale(), runtime.NumCPU(), Compress()) + path, err = library.Restore(holotreeBlueprint, []byte(common.ControllerIdentity()), []byte(common.HolotreeSpace)) + fail.On(err != nil, "Failed to restore blueprint %q, reason: %v", string(holotreeBlueprint), err) + journal.CurrentBuildEvent().RestoreComplete() + } else { + pretty.Progress(14, "Restoring space skipped.") + } + + externally, err = conda.ApplyExternallyManaged(path) + fail.Fast(err) + fail.Fast(CleanupHolotreeStage(tree)) + + return path, scorecard, nil +} + +func CleanupHolotreeStage(tree MutableLibrary) error { + common.TimelineBegin("holotree stage removal") + defer common.TimelineEnd() + location := tree.Stage() + common.Timeline("- removing %q", location) + return pathlib.TryRemoveAll("stage", location) +} + +func RecordEnvironment(tree MutableLibrary, blueprint []byte, force bool, scorecard common.Scorecard, puller CatalogPuller) (err error) { + defer fail.Around(&err) + + // following must be setup here + common.StageFolder = tree.Stage() + backup := common.Liveonly + common.Liveonly = true + defer func() { + common.Liveonly = backup + }() + + common.Debug("Holotree stage is %q.", tree.Stage()) + exists := tree.HasBlueprint(blueprint) + common.Debug("Has blueprint environment: %v", exists) + + conda.LogUnifiedEnvironment(blueprint) + + if force || !exists { + common.FreshlyBuildEnvironment = true + remoteOrigin := common.RccRemoteOrigin() + if len(remoteOrigin) > 0 { + pretty.Progress(3, "Fill hololib from RCC_REMOTE_ORIGIN.") + hash := common.BlueprintHash(blueprint) + catalog := CatalogName(hash) + err = puller(remoteOrigin, catalog, false) + if err != nil { + pretty.Warning("Failed to pull %q from %q, reason: %v", catalog, remoteOrigin, err) + } else { + return nil + } + exists = tree.HasBlueprint(blueprint) + } else { + pretty.Progress(3, "Fill hololib from RCC_REMOTE_ORIGIN skipped. RCC_REMOTE_ORIGIN was not defined.") + } + pretty.Progress(4, "Cleanup holotree stage for fresh install.") + fail.On(settings.Global.NoBuild(), "Building new holotree environment is blocked by settings, and could not be found from hololib cache!") + err = CleanupHolotreeStage(tree) + fail.On(err != nil, "Failed to clean stage, reason %v.", err) + journal.CurrentBuildEvent().PrepareComplete() + + err = os.MkdirAll(tree.Stage(), 0o755) + fail.On(err != nil, "Failed to create stage, reason %v.", err) + + pretty.Progress(5, "Build environment into holotree stage %q.", tree.Stage()) + identityfile := filepath.Join(tree.Stage(), "identity.yaml") + err = os.WriteFile(identityfile, blueprint, 0o644) + fail.On(err != nil, "Failed to save %q, reason %w.", identityfile, err) + + skip := conda.SkipNoLayers + if !force { + pretty.Progress(6, "Restore partial environment into holotree stage %q.", tree.Stage()) + skip = RestoreLayersTo(tree, identityfile, tree.Stage()) + } else { + pretty.Progress(6, "Restore partial environment skipped. Force used.") + } + + err = os.WriteFile(identityfile, blueprint, 0o644) + fail.On(err != nil, "Failed to save %q, reason %w.", identityfile, err) + + err = conda.LegacyEnvironment(tree, force, skip, identityfile) + fail.On(err != nil, "Failed to create environment, reason %w.", err) + + scorecard.Midpoint() + + pretty.Progress(13, "Record holotree stage to hololib [with %d workers on %d CPUs].", anywork.Scale(), runtime.NumCPU()) + err = tree.Record(blueprint) + fail.On(err != nil, "Failed to record blueprint %q, reason: %w", string(blueprint), err) + journal.CurrentBuildEvent().RecordComplete() + } + + return nil +} + +func RestoreLayersTo(tree MutableLibrary, identityfile string, targetDir string) conda.SkipLayer { + config, err := conda.ReadPackageCondaYaml(identityfile) + if err != nil { + return conda.SkipNoLayers + } + + layers := config.AsLayers() + mambaLayer := []byte(layers[0]) + pipLayer := []byte(layers[1]) + base := filepath.Base(targetDir) + if tree.HasBlueprint(pipLayer) { + _, err = tree.RestoreTo(pipLayer, base, common.ControllerIdentity(), common.HolotreeSpace, true) + if err == nil { + return conda.SkipPipLayer + } + } + if tree.HasBlueprint(mambaLayer) { + _, err = tree.RestoreTo(mambaLayer, base, common.ControllerIdentity(), common.HolotreeSpace, true) + if err == nil { + return conda.SkipMicromambaLayer + } + } + return conda.SkipNoLayers +} + +func RobotBlueprints(userBlueprints []string, packfile string) (robot.Robot, []string) { + var err error + var config robot.Robot + + blueprints := make([]string, 0, len(userBlueprints)+2) + + if len(packfile) > 0 { + config, err = robot.LoadRobotYaml(packfile, false) + if err == nil { + blueprints = append(blueprints, config.CondaConfigFile()) + } + } + + return config, append(blueprints, userBlueprints...) +} + +func ComposeFinalBlueprint(userFiles []string, packfile string) (config robot.Robot, blueprint []byte, err error) { + defer fail.Around(&err) + + var left, right *conda.Environment + + config, filenames := RobotBlueprints(userFiles, packfile) + + for _, filename := range filenames { + left = right + right, err = conda.ReadPackageCondaYaml(filename) + fail.On(err != nil, "Failure: %v", err) + if left == nil { + continue + } + right, err = left.Merge(right) + fail.On(err != nil, "Failure: %v", err) + } + fail.On(right == nil, "Missing environment specification(s).") + content, err := right.AsYaml() + fail.On(err != nil, "YAML error: %v", err) + blueprint = []byte(strings.TrimSpace(content)) + if !right.IsCacheable() { + fingerprint := common.BlueprintHash(blueprint) + pretty.Warning("Holotree blueprint %q is not publicly cacheable. Use `rcc robot diagnostics` to find out more.", fingerprint) + } + return config, blueprint, nil +} diff --git a/htfs/delegates.go b/htfs/delegates.go new file mode 100644 index 00000000..998eac92 --- /dev/null +++ b/htfs/delegates.go @@ -0,0 +1,37 @@ +package htfs + +import ( + "compress/gzip" + "io" + "os" + + "github.com/robocorp/rcc/fail" +) + +func gzDelegateOpen(filename string, ungzip bool) (readable io.Reader, closer Closer, err error) { + defer fail.Around(&err) + + source, err := os.Open(filename) + fail.On(err != nil, "Failed to open %q -> %v", filename, err) + + var reader io.ReadCloser + reader, err = gzip.NewReader(source) + if err != nil || !ungzip { + _, err = source.Seek(0, 0) + fail.On(err != nil, "Failed to seek %q -> %v", filename, err) + reader = source + closer = func() error { + return source.Close() + } + } else { + closer = func() error { + reader.Close() + return source.Close() + } + } + return reader, closer, nil +} + +func delegateOpen(it MutableLibrary, digest string, ungzip bool) (readable io.Reader, closer Closer, err error) { + return gzDelegateOpen(it.ExactLocation(digest), ungzip) +} diff --git a/htfs/directory.go b/htfs/directory.go new file mode 100644 index 00000000..89ed7ce9 --- /dev/null +++ b/htfs/directory.go @@ -0,0 +1,485 @@ +package htfs + +import ( + "bytes" + "compress/gzip" + "encoding/json" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "strings" + + "github.com/robocorp/rcc/anywork" + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/fail" + "github.com/robocorp/rcc/pathlib" + "github.com/robocorp/rcc/set" +) + +var ( + killfile map[string]bool +) + +func init() { + killfile = make(map[string]bool) + killfile["__MACOSX"] = true + killfile["__pycache__"] = true + killfile[".pyc"] = true + killfile[".git"] = true + killfile[".hg"] = true + killfile[".svn"] = true + killfile[".gitignore"] = true + + if !common.WarrantyVoided() { + pathlib.MakeSharedDir(common.Product.HoloLocation()) + pathlib.MakeSharedDir(common.HololibCatalogLocation()) + pathlib.MakeSharedDir(common.HololibLibraryLocation()) + pathlib.MakeSharedDir(common.HololibUsageLocation()) + pathlib.MakeSharedDir(common.HololibPids()) + } +} + +type ( + Filetask func(string, *File) anywork.Work + Dirtask func(string, *Dir) anywork.Work + Treetop func(string, *Dir) error + + Info struct { + RccVersion string `json:"rcc"` + Identity string `json:"identity"` + Path string `json:"path"` + Controller string `json:"controller"` + Space string `json:"space"` + Platform string `json:"platform"` + Blueprint string `json:"blueprint"` + } + + Root struct { + *Info + Lifted bool `json:"lifted"` + Tree *Dir `json:"tree"` + source string + } + + Roots []*Root + Dir struct { + Name string `json:"name"` + Symlink string `json:"symlink,omitempty"` + Mode fs.FileMode `json:"mode"` + Dirs map[string]*Dir `json:"subdirs"` + Files map[string]*File `json:"files"` + Shadow bool `json:"shadow,omitempty"` + } + + File struct { + Name string `json:"name"` + Symlink string `json:"symlink,omitempty"` + Size int64 `json:"size"` + Mode fs.FileMode `json:"mode"` + Digest string `json:"digest"` + Rewrite []int64 `json:"rewrite"` + } +) + +func (it *Info) AsJson() ([]byte, error) { + return json.MarshalIndent(it, "", " ") +} + +func (it *Info) saveAs(filename string) error { + content, err := it.AsJson() + if err != nil { + return err + } + sink, err := pathlib.Create(filename) + if err != nil { + return err + } + defer sink.Close() + defer sink.Sync() + _, err = sink.Write(content) + if err != nil { + return err + } + return nil +} + +func (it Roots) BaseFolders() []string { + result := []string{} + for _, root := range it { + result = append(result, filepath.Dir(root.Path)) + } + return set.Set(result) +} + +func (it Roots) Spaces() Roots { + roots := make(Roots, 0, 20) + for directory, metafile := range it.Spacemap() { + root, err := NewRoot(directory) + if err != nil { + continue + } + err = root.LoadFrom(metafile) + if err != nil { + continue + } + roots = append(roots, root) + } + return roots +} + +func (it Roots) Spacemap() map[string]string { + result := make(map[string]string) + for _, basedir := range it.BaseFolders() { + for _, metafile := range pathlib.Glob(basedir, "*.meta") { + result[metafile[:len(metafile)-5]] = metafile + } + } + return result +} + +func (it Roots) FindEnvironments(fragments []string) []string { + result := make([]string, 0, 10) + for directory, _ := range it.Spacemap() { + name := filepath.Base(directory) + for _, fragment := range fragments { + if strings.Contains(name, fragment) { + result = append(result, name) + } + } + } + return set.Set(result) +} + +func (it Roots) InstallationPlan(hash string) (string, bool) { + for _, directory := range it.BaseFolders() { + finalplan := filepath.Join(directory, hash, "rcc_plan.log") + if pathlib.IsFile(finalplan) { + return finalplan, true + } + } + return "", false +} + +func (it Roots) RemoveHolotreeSpace(label string) (err error) { + defer fail.Around(&err) + + for directory, metafile := range it.Spacemap() { + name := filepath.Base(directory) + if name != label { + continue + } + pathlib.TryRemove("metafile", metafile) + pathlib.TryRemove("lockfile", directory+".lck") + err = pathlib.TryRemoveAll("space", directory) + fail.On(err != nil, "Problem removing %q, reason: %v.", directory, err) + common.Timeline("removed holotree space %q", directory) + } + return nil +} + +func NewInfo(path string) (*Info, error) { + fullpath, err := pathlib.Abs(path) + if err != nil { + return nil, err + } + return &Info{ + RccVersion: common.Version, + Identity: filepath.Base(fullpath), + Path: fullpath, + Platform: common.Platform(), + }, nil +} + +func NewRoot(path string) (*Root, error) { + info, err := NewInfo(path) + if err != nil { + return nil, err + } + return &Root{ + Info: info, + Lifted: false, + Tree: newDir("", "", false), + source: info.Path, + }, nil +} + +func (it *Root) Top(count int) map[string]int64 { + target := make(map[string]int64) + it.Tree.fillSizes("", target) + sizes := set.Values(target) + total := len(sizes) + if total > count { + sizes = sizes[total-count:] + } + members := set.Membership(sizes) + result := make(map[string]int64) + for filename, size := range target { + if members[size] { + result[filename] = size + } + } + return result +} + +func (it *Root) Show(filename string) ([]byte, error) { + return it.Tree.Show(filepath.SplitList(filename), filename) +} + +func (it *Root) Source() string { + return it.source +} + +func (it *Root) Touch() { + touchUsedHash(it.Blueprint) +} + +func (it *Root) HolotreeBase() string { + return filepath.Dir(it.Path) +} + +func (it *Root) Signature() uint64 { + return common.Sipit([]byte(strings.ToLower(fmt.Sprintf("%s %q", it.Platform, it.Path)))) +} + +func (it *Root) Rewrite() []byte { + return []byte(it.Identity) +} + +func (it *Root) Relocate(target string) error { + locate := filepath.Dir(target) + if it.HolotreeBase() != locate { + return fmt.Errorf("Base directory mismatch: %q vs %q.", it.HolotreeBase(), locate) + } + basename := filepath.Base(target) + if len(it.Identity) != len(basename) { + return fmt.Errorf("Base name length mismatch: %q vs %q.", it.Identity, basename) + } + if len(it.Path) != len(target) { + return fmt.Errorf("Path length mismatch: %q vs %q.", it.Path, target) + } + it.Path = target + it.Identity = basename + return nil +} + +func (it *Root) Lift() error { + if it.Lifted { + return nil + } + it.Lifted = true + return it.Tree.Lift(it.Path) +} + +func (it *Root) Treetop(task Treetop) error { + common.TimelineBegin("holotree treetop sync start") + defer common.TimelineEnd() + err := task(it.Path, it.Tree) + if err != nil { + return err + } + return anywork.Sync() +} + +func (it *Root) Stats() (*TreeStats, error) { + task, stats := CalculateTreeStats() + err := it.AllDirs(task) + if err != nil { + return nil, err + } + return stats, nil +} + +func (it *Root) AllDirs(task Dirtask) error { + common.TimelineBegin("holotree dirs sync start") + defer common.TimelineEnd() + it.Tree.AllDirs(it.Path, task) + return anywork.Sync() +} + +func (it *Root) AllFiles(task Filetask) error { + common.TimelineBegin("holotree files sync start") + defer common.TimelineEnd() + it.Tree.AllFiles(it.Path, task) + return anywork.Sync() +} + +func (it *Root) AsJson() ([]byte, error) { + return json.MarshalIndent(it, "", " ") +} + +func (it *Root) SaveAs(filename string) error { + content, err := it.AsJson() + if err != nil { + return err + } + sink, err := pathlib.Create(filename) + if err != nil { + return err + } + defer sink.Close() + defer sink.Sync() + writer, err := gzip.NewWriterLevel(sink, gzip.BestSpeed) + if err != nil { + return err + } + defer writer.Close() + _, err = writer.Write(content) + if err != nil { + return err + } + return it.Info.saveAs(filename + ".info") +} + +func (it *Root) ReadFrom(source io.Reader) error { + decoder := json.NewDecoder(source) + return decoder.Decode(&it) +} + +func (it *Root) LoadFrom(filename string) error { + source, err := os.Open(filename) + if err != nil { + return err + } + defer source.Close() + reader, err := gzip.NewReader(source) + if err != nil { + return err + } + it.source = filename + defer common.Timeline("holotree catalog %q loaded", filename) + defer reader.Close() + return it.ReadFrom(reader) +} + +func showFile(filename string) (content []byte, err error) { + defer fail.Around(&err) + + reader, closer, err := gzDelegateOpen(filename, true) + fail.On(err != nil, "Failed to open %q, reason: %v", filename, err) + defer closer() + + sink := bytes.NewBuffer(nil) + _, err = io.Copy(sink, reader) + fail.On(err != nil, "Failed to read %q, reason: %v", filename, err) + return sink.Bytes(), nil +} + +func (it *Dir) fillSizes(prefix string, target map[string]int64) { + for filename, file := range it.Files { + target[filepath.Join(prefix, filename)] = file.Size + } + for dirname, dir := range it.Dirs { + dir.fillSizes(filepath.Join(prefix, dirname), target) + } +} + +func (it *Dir) Show(path []string, fullpath string) ([]byte, error) { + if len(path) > 1 { + subtree, ok := it.Dirs[path[0]] + if !ok { + return nil, fmt.Errorf("Not found: %s", fullpath) + } + return subtree.Show(path[1:], fullpath) + } + file, ok := it.Files[path[0]] + if !ok { + return nil, fmt.Errorf("Not found: %s", fullpath) + } + location := guessLocation(file.Digest) + rawfile := filepath.Join(common.HololibLibraryLocation(), location) + return showFile(rawfile) +} + +func (it *Dir) IsSymlink() bool { + return len(it.Symlink) > 0 +} + +func (it *Dir) AllDirs(path string, task Dirtask) { + for name, dir := range it.Dirs { + fullpath := filepath.Join(path, name) + dir.AllDirs(fullpath, task) + } + anywork.Backlog(task(path, it)) +} + +func (it *Dir) AllFiles(path string, task Filetask) { + for name, dir := range it.Dirs { + fullpath := filepath.Join(path, name) + dir.AllFiles(fullpath, task) + } + for name, file := range it.Files { + fullpath := filepath.Join(path, name) + anywork.Backlog(task(fullpath, file)) + } +} + +func (it *Dir) Lift(path string) error { + stat, err := os.Stat(path) + if err != nil { + return err + } + it.Mode = stat.Mode() + content, err := os.ReadDir(path) + if err != nil { + return err + } + shadow := it.Shadow || it.IsSymlink() + for _, part := range content { + if killfile[part.Name()] || killfile[filepath.Ext(part.Name())] { + continue + } + fullpath := filepath.Join(path, part.Name()) + // following must be done to get by symbolic links + info, err := os.Stat(fullpath) + if err != nil { + return err + } + symlink, _ := pathlib.Symlink(fullpath) + if info.IsDir() { + it.Dirs[part.Name()] = newDir(info.Name(), symlink, shadow) + continue + } + it.Files[part.Name()] = newFile(info, symlink) + } + for name, dir := range it.Dirs { + err = dir.Lift(filepath.Join(path, name)) + if err != nil { + return err + } + } + return nil +} + +func (it *File) IsSymlink() bool { + return len(it.Symlink) > 0 +} + +func (it *File) Match(info fs.FileInfo) bool { + name := it.Name == info.Name() + size := it.Size == info.Size() + mode := it.Mode == info.Mode() + return name && size && mode +} + +func newDir(name, symlink string, shadow bool) *Dir { + return &Dir{ + Name: name, + Symlink: symlink, + Dirs: make(map[string]*Dir), + Files: make(map[string]*File), + Shadow: shadow, + } +} + +func newFile(info fs.FileInfo, symlink string) *File { + return &File{ + Name: info.Name(), + Symlink: symlink, + Mode: info.Mode(), + Size: info.Size(), + Digest: "N/A", + Rewrite: make([]int64, 0), + } +} diff --git a/htfs/fs_test.go b/htfs/fs_test.go new file mode 100644 index 00000000..418d47b1 --- /dev/null +++ b/htfs/fs_test.go @@ -0,0 +1,74 @@ +package htfs_test + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/robocorp/rcc/common" + + "github.com/robocorp/rcc/hamlet" + "github.com/robocorp/rcc/htfs" +) + +func TestHTFSspecification(t *testing.T) { + must, wont := hamlet.Specifications(t) + + filename := filepath.Join(os.TempDir(), "htfs_test.json") + + fs, err := htfs.NewRoot("..") + must.Nil(err) + wont.Nil(fs) + wont.Nil(fs.Tree) + + must.Nil(fs.Lift()) + + content, err := fs.AsJson() + must.Nil(err) + must.True(len(content) > 50000) + + must.Nil(fs.SaveAs(filename)) + + reloaded, err := htfs.NewRoot(".") + must.Nil(err) + wont.Nil(reloaded) + before, err := reloaded.AsJson() + must.Nil(err) + must.True(len(before) < 300) + wont.Equal(fs.Path, reloaded.Path) + must.Nil(reloaded.LoadFrom(filename)) + after, err := reloaded.AsJson() + must.Nil(err) + must.Equal(len(after), len(content)) + must.Equal(fs.Path, reloaded.Path) +} + +// This test case depends on runtime.GOARCH being "amd64" - this is enforced +// when running unit tests with invoke, but if the test suite is run otherwise, +// for example directly from the IDE, GOARCH env variable needs to be set in order +// for this test to pass. +func TestZipLibrary(t *testing.T) { + must, wont := hamlet.Specifications(t) + + platform := common.Platform() + var zipFileName string + + switch { + case strings.Contains(platform, "linux"): + zipFileName = "simple_linux.zip" + case strings.Contains(platform, "darwin"): + zipFileName = "simple_darwin.zip" + case strings.Contains(platform, "windows"): + zipFileName = "simple_windows.zip" + } + + _, blueprint, err := htfs.ComposeFinalBlueprint([]string{"testdata/simple.yaml"}, "") + must.Nil(err) + wont.Nil(blueprint) + sut, err := htfs.ZipLibrary(fmt.Sprintf("testdata/%s", zipFileName)) + must.Nil(err) + wont.Nil(sut) + must.True(sut.HasBlueprint(blueprint)) +} diff --git a/htfs/functions.go b/htfs/functions.go new file mode 100644 index 00000000..3458fb57 --- /dev/null +++ b/htfs/functions.go @@ -0,0 +1,560 @@ +package htfs + +import ( + "compress/gzip" + "fmt" + "io" + "os" + "path/filepath" + "runtime" + "sync" + + "github.com/robocorp/rcc/anywork" + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/fail" + "github.com/robocorp/rcc/pathlib" +) + +func justFileExistCheck(location string, path, name, digest string) anywork.Work { + return func() { + if !pathlib.IsFile(location) { + fullpath := filepath.Join(path, name) + panic(fmt.Errorf("Content for %q [%s] is missing; hololib is broken, requires check!", fullpath, digest)) + } + } +} + +func CatalogCheck(library MutableLibrary, fs *Root) Treetop { + var tool Treetop + scheduled := make(map[string]bool) + tool = func(path string, it *Dir) error { + for name, file := range it.Files { + if !scheduled[file.Digest] { + anywork.Backlog(justFileExistCheck(library.ExactLocation(file.Digest), path, name, file.Digest)) + scheduled[file.Digest] = true + } + } + for name, subdir := range it.Dirs { + err := tool(filepath.Join(path, name), subdir) + if err != nil { + return err + } + } + return nil + } + return tool +} + +func DigestMapper(target map[string]string) Treetop { + var tool Treetop + tool = func(path string, it *Dir) error { + for name, subdir := range it.Dirs { + tool(filepath.Join(path, name), subdir) + } + for name, file := range it.Files { + target[file.Digest] = filepath.Join(path, name) + } + return nil + } + return tool +} + +func DigestRecorder(target map[string]string) Treetop { + var tool Treetop + tool = func(path string, it *Dir) error { + for name, subdir := range it.Dirs { + tool(filepath.Join(path, name), subdir) + } + for name, file := range it.Files { + target[filepath.Join(path, name)] = file.Digest + } + return nil + } + return tool +} + +func IntegrityCheck(result map[string]string, needed map[string]map[string]bool) Treetop { + var tool Treetop + tool = func(path string, it *Dir) error { + for name, subdir := range it.Dirs { + tool(filepath.Join(path, name), subdir) + } + for name, file := range it.Files { + if file.Name != file.Digest { + result[filepath.Join(path, name)] = file.Digest + } else { + delete(needed, file.Digest) + } + } + return nil + } + return tool +} + +func CheckHasher(known map[string]map[string]bool) Filetask { + return func(fullpath string, details *File) anywork.Work { + return func() { + _, ok := known[details.Name] + if !ok { + defer anywork.Backlog(RemoveFile(fullpath)) + } + source, err := os.Open(fullpath) + if err != nil { + anywork.Backlog(RemoveFile(fullpath)) + panic(fmt.Sprintf("Open[check] %q, reason: %v", fullpath, err)) + } + defer source.Close() + + var reader io.ReadCloser + reader, err = gzip.NewReader(source) + if err != nil { + _, err = source.Seek(0, 0) + fail.On(err != nil, "Failed to seek %q -> %v", fullpath, err) + reader = source + } + digest := common.NewDigester(Compress()) + _, err = io.Copy(digest, reader) + if err != nil { + anywork.Backlog(RemoveFile(fullpath)) + panic(fmt.Sprintf("Copy[check] %q, reason: %v", fullpath, err)) + } + details.Digest = fmt.Sprintf("%02x", digest.Sum(nil)) + } + } +} + +func Locator(seek string) Filetask { + return func(fullpath string, details *File) anywork.Work { + return func() { + source, err := os.Open(fullpath) + if err != nil { + panic(fmt.Sprintf("Open[Locator] %q, reason: %v", fullpath, err)) + } + defer source.Close() + digest := common.NewDigester(Compress()) + locator := RelocateWriter(digest, seek) + _, err = io.Copy(locator, source) + if err != nil { + panic(fmt.Sprintf("Copy[Locator] %q, reason: %v", fullpath, err)) + } + details.Rewrite = locator.Locations() + details.Digest = fmt.Sprintf("%02x", digest.Sum(nil)) + } + } +} + +func MakeBranches(path string, it *Dir) error { + if it.Shadow || it.IsSymlink() { + return nil + } + if _, ok := pathlib.Symlink(path); ok { + os.Remove(path) + } + hasSymlinks := false +detector: + for _, subdir := range it.Dirs { + if subdir.IsSymlink() { + hasSymlinks = true + break detector + } + } + if hasSymlinks { + err := os.MkdirAll(path, 0o750) + if err != nil { + return err + } + } + for _, subdir := range it.Dirs { + err := MakeBranches(filepath.Join(path, subdir.Name), subdir) + if err != nil { + return err + } + } + if len(it.Dirs) == 0 { + err := os.MkdirAll(path, 0o750) + if err != nil { + return err + } + } + return os.Chtimes(path, motherTime, motherTime) +} + +func ScheduleLifters(library MutableLibrary, stats *stats) Treetop { + var scheduler Treetop + compress := Compress() + seen := make(map[string]bool) + scheduler = func(path string, it *Dir) error { + if it.IsSymlink() { + return nil + } + for name, subdir := range it.Dirs { + scheduler(filepath.Join(path, name), subdir) + } + for name, file := range it.Files { + if file.IsSymlink() { + stats.Link() + continue + } + if seen[file.Digest] { + common.Trace("LiftFile %s %q already scheduled.", file.Digest, name) + stats.Duplicate() + continue + } + seen[file.Digest] = true + directory := library.Location(file.Digest) + if !seen[directory] && !pathlib.IsDir(directory) { + pathlib.MakeSharedDir(directory) + } + seen[directory] = true + sinkpath := filepath.Join(directory, file.Digest) + ok := pathlib.IsFile(sinkpath) + stats.Dirty(!ok) + if ok { + continue + } + sourcepath := filepath.Join(path, name) + anywork.Backlog(LiftFile(sourcepath, sinkpath, compress)) + } + return nil + } + return scheduler +} + +func LiftFile(sourcename, sinkname string, compress bool) anywork.Work { + return func() { + source, err := os.Open(sourcename) + anywork.OnErrPanicCloseAll(err) + + defer source.Close() + partname := fmt.Sprintf("%s.part%s", sinkname, <-common.Identities) + defer os.Remove(partname) + sink, err := os.Create(partname) + anywork.OnErrPanicCloseAll(err) + + defer sink.Close() + + var writer io.WriteCloser + writer = sink + if compress { + writer, err = gzip.NewWriterLevel(sink, gzip.BestSpeed) + anywork.OnErrPanicCloseAll(err, sink) + } + + _, err = io.Copy(writer, source) + anywork.OnErrPanicCloseAll(err, sink) + + if compress { + anywork.OnErrPanicCloseAll(writer.Close(), sink) + } + + anywork.OnErrPanicCloseAll(sink.Close()) + + runtime.Gosched() + + anywork.OnErrPanicCloseAll(pathlib.TryRename("liftfile", partname, sinkname)) + pathlib.MakeSharedFile(sinkname) + } +} + +func DropFile(library Library, digest, sinkname string, details *File, rewrite []byte) anywork.Work { + return func() { + if details.IsSymlink() { + anywork.OnErrPanicCloseAll(restoreSymlink(details.Symlink, sinkname)) + return + } + reader, closer, err := library.Open(digest) + anywork.OnErrPanicCloseAll(err) + + defer closer() + partname := fmt.Sprintf("%s.part%s", sinkname, <-common.Identities) + defer os.Remove(partname) + sink, err := os.Create(partname) + anywork.OnErrPanicCloseAll(err) + + digester := common.NewDigester(Compress()) + many := io.MultiWriter(sink, digester) + + _, err = io.Copy(many, reader) + anywork.OnErrPanicCloseAll(err, sink) + + hexdigest := fmt.Sprintf("%02x", digester.Sum(nil)) + if digest != hexdigest { + err := fmt.Errorf("Corrupted hololib, expected %s, actual %s", digest, hexdigest) + anywork.OnErrPanicCloseAll(err, sink) + } + + for _, position := range details.Rewrite { + _, err = sink.Seek(position, 0) + if err != nil { + sink.Close() + panic(fmt.Sprintf("%v %d", err, position)) + } + _, err = sink.Write(rewrite) + anywork.OnErrPanicCloseAll(err, sink) + } + + anywork.OnErrPanicCloseAll(sink.Close()) + + anywork.OnErrPanicCloseAll(pathlib.TryRename("dropfile", partname, sinkname)) + + anywork.OnErrPanicCloseAll(os.Chmod(sinkname, details.Mode)) + anywork.OnErrPanicCloseAll(os.Chtimes(sinkname, motherTime, motherTime)) + } +} + +func RemoveFile(filename string) anywork.Work { + return func() { + anywork.OnErrPanicCloseAll(pathlib.TryRemove("file", filename)) + } +} + +func RemoveDirectory(dirname string) anywork.Work { + return func() { + anywork.OnErrPanicCloseAll(pathlib.TryRemoveAll("directory", dirname)) + } +} + +type TreeStats struct { + sync.Mutex + Directories uint64 + Files uint64 + Bytes uint64 + Identity string + Relocations uint64 +} + +func guessLocation(digest string) string { + return filepath.Join(digest[:2], digest[2:4], digest[4:6], digest) +} + +func CalculateTreeStats() (Dirtask, *TreeStats) { + result := &TreeStats{} + return func(path string, it *Dir) anywork.Work { + return func() { + result.Lock() + defer result.Unlock() + result.Directories += 1 + result.Files += uint64(len(it.Files)) + for _, file := range it.Files { + result.Bytes += uint64(file.Size) + if file.Name == "identity.yaml" { + result.Identity = guessLocation(file.Digest) + } + if len(file.Rewrite) > 0 { + result.Relocations += 1 + } + } + } + }, result +} + +func isCorrectSymlink(source, target string) bool { + old, ok := pathlib.Symlink(target) + return ok && old == source +} + +func restoreSymlink(source, target string) error { + if isCorrectSymlink(source, target) { + return nil + } + os.RemoveAll(target) + return os.Symlink(source, target) +} + +func RestoreDirectory(library Library, fs *Root, current map[string]string, stats *stats) Dirtask { + return func(path string, it *Dir) anywork.Work { + return func() { + if it.Shadow { + return + } + if it.IsSymlink() { + anywork.OnErrPanicCloseAll(restoreSymlink(it.Symlink, path)) + return + } + existingEntries, err := os.ReadDir(path) + anywork.OnErrPanicCloseAll(err) + files := make(map[string]bool) + for _, part := range existingEntries { + directpath := filepath.Join(path, part.Name()) + if part.IsDir() { + _, ok := it.Dirs[part.Name()] + if !ok { + common.Trace("* Holotree: remove extra directory %q", directpath) + anywork.Backlog(RemoveDirectory(directpath)) + } + stats.Dirty(!ok) + continue + } + link, ok := it.Dirs[part.Name()] + if ok && link.IsSymlink() { + stats.Link() + continue + } + files[part.Name()] = true + found, ok := it.Files[part.Name()] + if !ok { + common.Trace("* Holotree: remove extra file %q", directpath) + anywork.Backlog(RemoveFile(directpath)) + stats.Dirty(true) + continue + } + if found.IsSymlink() && isCorrectSymlink(found.Symlink, directpath) { + stats.Link() + continue + } + shadow, ok := current[directpath] + golden := !ok || found.Digest == shadow + info, err := part.Info() + anywork.OnErrPanicCloseAll(err) + ok = golden && found.Match(info) + stats.Dirty(!ok) + if !ok { + common.Trace("* Holotree: update changed file %q", directpath) + anywork.Backlog(DropFile(library, found.Digest, directpath, found, fs.Rewrite())) + } + } + for name, found := range it.Files { + directpath := filepath.Join(path, name) + _, seen := files[name] + if !seen { + stats.Dirty(true) + common.Trace("* Holotree: add missing file %q", directpath) + anywork.Backlog(DropFile(library, found.Digest, directpath, found, fs.Rewrite())) + } + } + } + } +} + +type Zipper interface { + Ignore(relativepath string) + Add(fullpath, relativepath string) error +} + +func ZipIgnore(library MutableLibrary, fs *Root, sink Zipper) Treetop { + var tool Treetop + baseline := common.HololibLocation() + tool = func(path string, it *Dir) (err error) { + defer fail.Around(&err) + + for _, file := range it.Files { + location := library.ExactLocation(file.Digest) + relative, err := filepath.Rel(baseline, location) + if err == nil { + sink.Ignore(relative) + } + } + for name, subdir := range it.Dirs { + err := tool(filepath.Join(path, name), subdir) + fail.On(err != nil, "%v", err) + } + return nil + } + return tool +} + +func ZipRoot(library MutableLibrary, fs *Root, sink Zipper) Treetop { + var tool Treetop + baseline := common.HololibLocation() + tool = func(path string, it *Dir) (err error) { + defer fail.Around(&err) + + for _, file := range it.Files { + location := library.ExactLocation(file.Digest) + relative, err := filepath.Rel(baseline, location) + fail.On(err != nil, "Relative path error: %s -> %s -> %v", baseline, location, err) + err = sink.Add(location, relative) + fail.On(err != nil, "%v", err) + } + for name, subdir := range it.Dirs { + err := tool(filepath.Join(path, name), subdir) + fail.On(err != nil, "%v", err) + } + return nil + } + return tool +} + +func LoadHololibHashes() (map[string]map[string]bool, map[string]map[string]bool) { + catalogs, roots := LoadCatalogs() + slots := make([]map[string]string, len(roots)) + for at, root := range roots { + anywork.Backlog(DigestLoader(root, at, slots)) + } + result := make(map[string]map[string]bool) + needed := make(map[string]map[string]bool) + runtime.Gosched() + anywork.Sync() + for at, slot := range slots { + catalog := catalogs[at] + for k, _ := range slot { + who, ok := needed[k] + if !ok { + who = make(map[string]bool) + needed[k] = who + } + who[catalog] = true + found, ok := result[k] + if !ok { + found = make(map[string]bool) + result[k] = found + } + found[catalog] = true + } + } + return result, needed +} + +func DigestLoader(root *Root, at int, slots []map[string]string) anywork.Work { + return func() { + collector := make(map[string]string) + task := DigestMapper(collector) + err := task(root.Path, root.Tree) + if err != nil { + panic(fmt.Sprintf("Collecting dir %q, reason: %v", root.Path, err)) + } + slots[at] = collector + common.Trace("Root %q loaded.", root.Path) + } +} + +func ignoreFailedCatalogs(suspects Roots) Roots { + roots := make(Roots, 0, len(suspects)) + for _, root := range suspects { + if root != nil { + roots = append(roots, root) + } + } + return roots +} + +func LoadCatalogs() ([]string, Roots) { + common.TimelineBegin("catalog load start") + defer common.TimelineEnd() + catalogs := CatalogNames() + roots := make(Roots, len(catalogs)) + for at, catalog := range catalogs { + fullpath := filepath.Join(common.HololibCatalogLocation(), catalog) + anywork.Backlog(CatalogLoader(fullpath, at, roots)) + catalogs[at] = fullpath + } + runtime.Gosched() + anywork.Sync() + return catalogs, ignoreFailedCatalogs(roots) +} + +func CatalogLoader(catalog string, at int, roots Roots) anywork.Work { + return func() { + tempdir := filepath.Join(common.ProductTemp(), "shadow") + shadow, err := NewRoot(tempdir) + if err != nil { + panic(fmt.Sprintf("Temp dir %q, reason: %v", tempdir, err)) + } + err = shadow.LoadFrom(catalog) + if err != nil { + panic(fmt.Sprintf("Load %q, reason: %v", catalog, err)) + } + roots[at] = shadow + common.Trace("Catalog %q loaded.", catalog) + } +} diff --git a/htfs/library.go b/htfs/library.go new file mode 100644 index 00000000..49c41713 --- /dev/null +++ b/htfs/library.go @@ -0,0 +1,478 @@ +package htfs + +import ( + "archive/zip" + "fmt" + "io" + "os" + "path/filepath" + "runtime" + "strings" + "sync" + "time" + + "github.com/robocorp/rcc/cloud" + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/fail" + "github.com/robocorp/rcc/journal" + "github.com/robocorp/rcc/pathlib" + "github.com/robocorp/rcc/pretty" + "github.com/robocorp/rcc/set" +) + +const ( + epoc = 1610000000 +) + +var ( + motherTime = time.Unix(epoc, 0) +) + +type stats struct { + sync.Mutex + total uint64 + dirty uint64 + links uint64 + duplicate uint64 +} + +func (it *stats) Dirtyness() float64 { + it.Lock() + defer it.Unlock() + + dirtyness := (1000 * it.dirty) / it.total + return float64(dirtyness) / 10.0 +} + +func (it *stats) Duplicate() { + it.Lock() + defer it.Unlock() + + it.total++ + it.duplicate++ +} + +func (it *stats) Link() { + it.Lock() + defer it.Unlock() + + it.total++ + it.links++ +} + +func (it *stats) Dirty(dirty bool) { + it.Lock() + defer it.Unlock() + + it.total++ + if dirty { + it.dirty++ + } +} + +type Closer func() error + +type Library interface { + ValidateBlueprint([]byte) error + HasBlueprint([]byte) bool + Open(string) (io.Reader, Closer, error) + WarrantyVoidedDir([]byte, []byte) string + TargetDir([]byte, []byte, []byte) (string, error) + Restore([]byte, []byte, []byte) (string, error) + RestoreTo([]byte, string, string, string, bool) (string, error) +} + +type MutableLibrary interface { + Library + + Identity() string + ExactLocation(string) string + Export([]string, []string, string) error + Remove([]string) error + Location(string) string + Record([]byte) error + Stage() string + CatalogPath(string) string + WriteIdentity([]byte) error +} + +type hololib struct { + identity uint64 + basedir string + queryCache map[string]bool +} + +func (it *hololib) Open(digest string) (readable io.Reader, closer Closer, err error) { + return delegateOpen(it, digest, Compress()) +} + +func (it *hololib) Location(digest string) string { + return filepath.Join(common.HololibLibraryLocation(), digest[:2], digest[2:4], digest[4:6]) +} + +func ExactDefaultLocation(digest string) string { + return filepath.Join(common.HololibLibraryLocation(), digest[:2], digest[2:4], digest[4:6], digest) +} + +func RelativeDefaultLocation(digest string) string { + location := ExactDefaultLocation(digest) + relative, _ := filepath.Rel(common.HololibLocation(), location) + return relative +} + +func (it *hololib) ExactLocation(digest string) string { + return ExactDefaultLocation(digest) +} + +func (it *hololib) Identity() string { + suffix := fmt.Sprintf("%016x", it.identity) + return fmt.Sprintf("h%s_%st", common.UserHomeIdentity(), suffix[:14]) +} + +func (it *hololib) WriteIdentity(yaml []byte) error { + markerFile := filepath.Join(it.Stage(), "identity.yaml") + return pathlib.WriteFile(markerFile, yaml, 0o644) +} + +func (it *hololib) Stage() string { + stage := filepath.Join(common.HolotreeLocation(), it.Identity()) + err := os.MkdirAll(stage, 0o755) + if err != nil { + panic(err) + } + return stage +} + +type zipseen struct { + *zip.Writer + seen map[string]bool +} + +func (it zipseen) Ignore(relativepath string) { + it.seen[relativepath] = true +} + +func (it zipseen) Add(fullpath, relativepath string) (err error) { + defer fail.Around(&err) + + if it.seen[relativepath] { + return nil + } + it.seen[relativepath] = true + + source, err := os.Open(fullpath) + fail.On(err != nil, "Could not open: %q -> %v", fullpath, err) + defer source.Close() + target, err := it.Create(relativepath) + fail.On(err != nil, "Could not create: %q -> %v", relativepath, err) + _, err = io.Copy(target, source) + fail.On(err != nil, "Copy failure: %q -> %q -> %v", fullpath, relativepath, err) + return nil +} + +func (it *hololib) Remove(catalogs []string) (err error) { + defer fail.Around(&err) + + common.TimelineBegin("holotree remove start") + defer common.TimelineEnd() + + for _, name := range catalogs { + catalog := filepath.Join(common.HololibCatalogLocation(), name) + if !pathlib.IsFile(catalog) { + pretty.Warning("Catalog %s (%s) is not a file! Ignored!", name, catalog) + continue + } + err := os.Remove(catalog) + fail.On(err != nil, "Could not remove catalog %s [filename: %q]", name, catalog) + } + return nil +} + +func (it *hololib) Export(catalogs, known []string, archive string) (err error) { + defer fail.Around(&err) + + common.TimelineBegin("holotree export start") + defer common.TimelineEnd() + + handle, err := pathlib.Create(archive) + fail.On(err != nil, "Could not create archive %q.", archive) + writer := zip.NewWriter(handle) + defer writer.Close() + + zipper := &zipseen{ + writer, + make(map[string]bool), + } + + exported := false + + for _, name := range known { + catalog := filepath.Join(common.HololibCatalogLocation(), name) + fs, err := NewRoot(".") + fail.On(err != nil, "Could not create root location -> %v.", err) + err = fs.LoadFrom(catalog) + if err != nil { + continue + } + err = fs.Treetop(ZipIgnore(it, fs, zipper)) + fail.On(err != nil, "Could not ignore catalog %s -> %v.", catalog, err) + } + + for _, name := range catalogs { + catalog := filepath.Join(common.HololibCatalogLocation(), name) + + fs, err := NewRoot(".") + fail.On(err != nil, "Could not create root location -> %v.", err) + err = fs.LoadFrom(catalog) + fail.On(err != nil, "Could not load catalog from %s -> %v.", catalog, err) + err = fs.Treetop(ZipRoot(it, fs, zipper)) + fail.On(err != nil, "Could not zip catalog %s -> %v.", catalog, err) + + relative, err := filepath.Rel(common.HololibLocation(), catalog) + fail.On(err != nil, "Could not get relative location for catalog -> %v.", err) + err = zipper.Add(catalog, relative) + fail.On(err != nil, "Could not add catalog to zip -> %v.", err) + + exported = true + } + fail.On(!exported, "None of given catalogs were available for export!") + return nil +} + +func (it *hololib) Record(blueprint []byte) error { + defer common.Stopwatch("Holotree recording took:").Debug() + err := it.WriteIdentity(blueprint) + if err != nil { + return err + } + key := common.BlueprintHash(blueprint) + common.TimelineBegin("holotree record start %s", key) + defer common.TimelineEnd() + fs, err := NewRoot(it.Stage()) + if err != nil { + return err + } + err = fs.Lift() + if err != nil { + return err + } + common.Timeline("holotree (re)locator start") + err = fs.AllFiles(Locator(it.Identity())) + if err != nil { + return err + } + common.Timeline("holotree (re)locator done") + fs.Blueprint = key + catalog := it.CatalogPath(key) + err = fs.SaveAs(catalog) + if err != nil { + return err + } + score := &stats{} + common.Timeline("holotree lift start %q", catalog) + err = fs.Treetop(ScheduleLifters(it, score)) + common.Timeline("holotree lift done") + defer common.Timeline("- new %d/%d (duplicate: %d, links: %d)", score.dirty, score.total, score.duplicate, score.links) + common.Debug("Holotree new workload: %d/%d\n", score.dirty, score.total) + return err +} + +func CatalogName(key string) string { + return fmt.Sprintf("%sv12.%s", key, common.Platform()) +} + +func (it *hololib) CatalogPath(key string) string { + return filepath.Join(common.HololibCatalogLocation(), CatalogName(key)) +} + +func (it *hololib) ValidateBlueprint(blueprint []byte) error { + return nil +} + +func Compress() bool { + return !pathlib.IsFile(common.HololibCompressMarker()) +} + +func (it *hololib) HasBlueprint(blueprint []byte) bool { + key := common.BlueprintHash(blueprint) + found, ok := it.queryCache[key] + if !ok { + found = it.queryBlueprint(key) + it.queryCache[key] = found + } + return found +} + +func (it *hololib) queryBlueprint(key string) bool { + common.Timeline("holotree blueprint query") + catalog := it.CatalogPath(key) + if !pathlib.IsFile(catalog) { + return false + } + tempdir := filepath.Join(common.ProductTemp(), key) + shadow, err := NewRoot(tempdir) + if err != nil { + return false + } + err = shadow.LoadFrom(catalog) + if err != nil { + common.Debug("Catalog load failed, reason: %v", err) + return false + } + common.TimelineBegin("holotree content check start") + err = shadow.Treetop(CatalogCheck(it, shadow)) + common.TimelineEnd() + if err != nil { + cloud.InternalBackgroundMetric(common.ControllerIdentity(), "rcc.holotree.catalog.failure", common.Version) + common.Debug("Catalog check failed, reason: %v", err) + return false + } + return pathlib.IsFile(catalog) +} + +func CatalogNames() []string { + result := make([]string, 0, 10) + for _, catalog := range pathlib.Glob(common.HololibCatalogLocation(), "[0-9a-f]*v12.*") { + if filepath.Ext(catalog) != ".info" { + result = append(result, filepath.Base(catalog)) + } + } + return set.Set(result) +} + +func ControllerSpaceName(client, tag []byte) string { + prefix := common.Textual(common.Sipit(client), 7) + suffix := common.Textual(common.Sipit(tag), 8) + return common.UserHomeIdentity() + "_" + prefix + "_" + suffix +} + +func touchUsedHash(hash string) { + filename := fmt.Sprintf("%s.%s", hash, common.UserHomeIdentity()) + fullpath := filepath.Join(common.HololibUsageLocation(), filename) + pathlib.ForceTouchWhen(fullpath, pretty.ProgressMark) +} + +func (it *hololib) WarrantyVoidedDir(controller, space []byte) string { + return filepath.Join(common.HolotreeLocation(), ControllerSpaceName(controller, space)) +} + +func (it *hololib) TargetDir(blueprint, controller, space []byte) (result string, err error) { + defer fail.Around(&err) + key := common.BlueprintHash(blueprint) + catalog := it.CatalogPath(key) + fs, err := NewRoot(it.Stage()) + fail.On(err != nil, "Failed to create stage -> %v", err) + name := ControllerSpaceName(controller, space) + err = fs.LoadFrom(catalog) + if err != nil { + return filepath.Join(common.HolotreeLocation(), name), nil + } + return filepath.Join(fs.HolotreeBase(), name), nil +} + +func UserHolotreeLockfile() string { + name := ControllerSpaceName([]byte(common.ControllerIdentity()), []byte(common.HolotreeSpace)) + return filepath.Join(common.HolotreeLocation(), fmt.Sprintf("%s.lck", name)) +} + +func (it *hololib) Restore(blueprint, client, tag []byte) (result string, err error) { + return it.RestoreTo(blueprint, ControllerSpaceName(client, tag), string(client), string(tag), false) +} + +func (it *hololib) RestoreTo(blueprint []byte, label, controller, space string, partial bool) (result string, err error) { + defer fail.Around(&err) + defer common.Stopwatch("Holotree restore took:").Debug() + + key := common.BlueprintHash(blueprint) + catalog := it.CatalogPath(key) + common.TimelineBegin("holotree space restore start [%s]", key) + defer common.TimelineEnd() + fs, err := NewRoot(it.Stage()) + fail.On(err != nil, "Failed to create stage -> %v", err) + err = fs.LoadFrom(catalog) + fail.On(err != nil, "Failed to load catalog %s -> %v", catalog, err) + targetdir := filepath.Join(fs.HolotreeBase(), label) + metafile := fmt.Sprintf("%s.meta", targetdir) + lockfile := fmt.Sprintf("%s.lck", targetdir) + completed := pathlib.LockWaitMessage(lockfile, "Serialized holotree restore [holotree base lock]") + locker, err := pathlib.Locker(lockfile, 30000, common.SharedHolotree) + completed() + fail.On(err != nil, "Could not get lock for %s. Quiting.", targetdir) + defer locker.Release() + journal.Post("space-used", metafile, "normal holotree with blueprint %s from %s", key, catalog) + currentstate := make(map[string]string) + mode := fmt.Sprintf("new space for %q", key) + shadow, err := NewRoot(targetdir) + if err == nil { + err = shadow.LoadFrom(metafile) + } + if err == nil { + if key == shadow.Blueprint { + mode = fmt.Sprintf("cleaned up space for %q", key) + } else { + mode = fmt.Sprintf("coverted space from %q to %q", shadow.Blueprint, key) + } + common.TimelineBegin("holotree digest start [%q -> %q]", shadow.Blueprint, key) + shadow.Treetop(DigestRecorder(currentstate)) + common.TimelineEnd() + } + common.Timeline("mode: %s", mode) + common.Debug("Holotree operating mode is: %s", mode) + err = fs.Relocate(targetdir) + fail.On(err != nil, "Failed to relocate %s -> %v", targetdir, err) + common.TimelineBegin("holotree make branches start") + err = fs.Treetop(MakeBranches) + common.TimelineEnd() + fail.On(err != nil, "Failed to make branches -> %v", err) + score := &stats{} + common.TimelineBegin("holotree restore start") + err = fs.AllDirs(RestoreDirectory(it, fs, currentstate, score)) + fail.On(err != nil, "Failed to restore directories -> %v", err) + common.TimelineEnd() + defer common.Timeline("- dirty %d/%d (duplicate: %d, links: %d)", score.dirty, score.total, score.duplicate, score.links) + common.Debug("Holotree dirty workload: %d/%d\n", score.dirty, score.total) + journal.CurrentBuildEvent().Dirty(score.Dirtyness()) + fs.Controller = controller + fs.Space = space + err = fs.SaveAs(metafile) + fail.On(err != nil, "Failed to save metafile %q -> %v", metafile, err) + pathlib.TouchWhen(catalog, time.Now()) + planfile := filepath.Join(targetdir, "rcc_plan.log") + if !partial && pathlib.FileExist(planfile) { + common.Log("%sInstallation plan is: %v%s", pretty.Yellow, planfile, pretty.Reset) + } + identityfile := filepath.Join(targetdir, "identity.yaml") + if !partial && pathlib.FileExist(identityfile) { + common.Log("%sEnvironment configuration descriptor is: %v%s", pretty.Yellow, identityfile, pretty.Reset) + } + touchUsedHash(key) + return targetdir, nil +} + +func makedirs(prefix string, suffixes ...string) error { + if common.Liveonly { + return nil + } + for _, suffix := range suffixes { + fullpath := filepath.Join(prefix, suffix) + _, err := pathlib.MakeSharedDir(fullpath) + if err != nil { + return err + } + } + return nil +} + +func New() (MutableLibrary, error) { + err := makedirs(common.HololibLocation(), "library", "catalog") + if err != nil { + return nil, err + } + basedir := common.Product.Home() + identity := strings.ToLower(fmt.Sprintf("%s %s", runtime.GOOS, runtime.GOARCH)) + return &hololib{ + identity: common.Sipit([]byte(identity)), + basedir: basedir, + queryCache: make(map[string]bool), + }, nil +} diff --git a/htfs/relocator.go b/htfs/relocator.go new file mode 100644 index 00000000..8d3ca1d6 --- /dev/null +++ b/htfs/relocator.go @@ -0,0 +1,82 @@ +package htfs + +import ( + "bytes" + "io" +) + +type ( + WriteLocator interface { + io.Writer + Locations() []int64 + } + + simple struct { + windowsize int + window []byte + trigger byte + needle []byte + history int64 + delegate io.Writer + found []int64 + } +) + +func RelocateWriter(delegate io.Writer, needle string) WriteLocator { + blob := []byte(needle) + windowsize := len(blob) + result := &simple{ + windowsize: windowsize, + window: make([]byte, 0, 8192+windowsize), + trigger: blob[windowsize-1], + needle: blob, + history: 0, + delegate: delegate, + found: make([]int64, 0, 20), + } + return result +} + +func (it *simple) trimWindow() { + shift := len(it.window) - it.windowsize + if shift > 0 { + copy(it.window, it.window[shift:]) + it.window = it.window[:it.windowsize] + } +} + +func (it *simple) Write(payload []byte) (int, error) { + pending := len(it.window) + it.window = append(it.window, payload...) + defer it.trimWindow() + + shift, view, trigger, limit := 0, it.window, it.trigger, it.windowsize +search: + for limit < len(view) { + found := bytes.IndexByte(view, trigger) + if found < 0 { + break search + } + head := view[:found+1] + view = view[found+1:] + end := shift + found + 1 - pending + start := end - limit + headsize := len(head) + if limit <= headsize { + candidate := head[headsize-limit:] + relation := bytes.Compare(it.needle, candidate) + if relation == 0 { + it.found = append(it.found, it.history+int64(start)) + } + } + shift += len(head) + } + + // seek here when found, append to it.found + it.history += int64(len(payload)) + return it.delegate.Write(payload) +} + +func (it *simple) Locations() []int64 { + return it.found +} diff --git a/htfs/relocator_test.go b/htfs/relocator_test.go new file mode 100644 index 00000000..25ee5a68 --- /dev/null +++ b/htfs/relocator_test.go @@ -0,0 +1,48 @@ +package htfs_test + +import ( + "bytes" + "fmt" + "testing" + + "github.com/robocorp/rcc/hamlet" + "github.com/robocorp/rcc/htfs" +) + +func TestBasics(t *testing.T) { + must, wont := hamlet.Specifications(t) + + sut := "simple text" + must.Equal(11, len(sut)) + wont.Panic(func() { + fmt.Sprintf("%q", sut[:11]) + }) + wont.Panic(func() { + fmt.Sprintf("%q", sut[11:]) + }) + must.Panic(func() { + fmt.Sprintf("%q", sut[:12]) + }) + must.Panic(func() { + fmt.Sprintf("%q", sut[12:]) + }) + must.Equal(sut, sut[:len(sut)]) + must.Equal("", sut[len(sut):]) +} + +func TestUsingNonhashingWorks(t *testing.T) { + must, wont := hamlet.Specifications(t) + + sink := bytes.NewBuffer(nil) + sut := htfs.RelocateWriter(sink, "loha") + + wont.Nil(sut) + size, err := sut.Write([]byte("O aloha! Aloha, Holoham!")) + must.Nil(err) + must.Equal(24, size) + must.Equal([]int64{3, 10, 18}, sut.Locations()) + size, err = sut.Write([]byte("O aloha! Aloha, Holoham!")) + must.Nil(err) + must.Equal(24, size) + must.Equal([]int64{3, 10, 18, 27, 34, 42}, sut.Locations()) +} diff --git a/htfs/testdata/.gitignore b/htfs/testdata/.gitignore new file mode 100644 index 00000000..fe22226a --- /dev/null +++ b/htfs/testdata/.gitignore @@ -0,0 +1,3 @@ +!simple_linux.zip +!simple_darwin.zip +!simple_windows.zip diff --git a/htfs/testdata/simple.yaml b/htfs/testdata/simple.yaml new file mode 100644 index 00000000..9c46ef58 --- /dev/null +++ b/htfs/testdata/simple.yaml @@ -0,0 +1,4 @@ +channels: + - conda-forge +dependencies: + - ca-certificates diff --git a/htfs/testdata/simple_darwin.zip b/htfs/testdata/simple_darwin.zip new file mode 100644 index 00000000..2c2f5613 Binary files /dev/null and b/htfs/testdata/simple_darwin.zip differ diff --git a/htfs/testdata/simple_linux.zip b/htfs/testdata/simple_linux.zip new file mode 100644 index 00000000..5f747b1f Binary files /dev/null and b/htfs/testdata/simple_linux.zip differ diff --git a/htfs/testdata/simple_windows.zip b/htfs/testdata/simple_windows.zip new file mode 100644 index 00000000..c28c63f5 Binary files /dev/null and b/htfs/testdata/simple_windows.zip differ diff --git a/htfs/unmanaged.go b/htfs/unmanaged.go new file mode 100644 index 00000000..e594ed54 --- /dev/null +++ b/htfs/unmanaged.go @@ -0,0 +1,146 @@ +package htfs + +import ( + "fmt" + "io" + "path/filepath" + + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/pathlib" + "github.com/robocorp/rcc/pretty" +) + +type unmanaged struct { + delegate MutableLibrary + path string + resolved bool + protected bool +} + +func Unmanaged(core MutableLibrary) MutableLibrary { + return &unmanaged{ + delegate: core, + path: "", + resolved: false, + protected: false, + } +} + +func (it *unmanaged) Identity() string { + return it.delegate.Identity() +} + +func (it *unmanaged) Stage() string { + return it.delegate.Stage() +} + +func (it *unmanaged) WriteIdentity([]byte) error { + return fmt.Errorf("Not supported yet on virtual holotree.") +} + +func (it *unmanaged) CatalogPath(key string) string { + return "Unmanaged Does Not Support Catalog Path Request" +} + +func (it *unmanaged) Remove([]string) error { + return fmt.Errorf("Not supported yet on unmanaged holotree.") +} + +func (it *unmanaged) Export([]string, []string, string) error { + return fmt.Errorf("Not supported yet on unmanaged holotree.") +} + +func (it *unmanaged) resolve(blueprint []byte) error { + if it.resolved { + return nil + } + defer common.Log("%sThis is unmanaged holotree space, checking suitability for blueprint: %v%s", pretty.Magenta, common.BlueprintHash(blueprint), pretty.Reset) + controller := []byte(common.ControllerIdentity()) + space := []byte(common.HolotreeSpace) + path, err := it.TargetDir(blueprint, controller, space) + if err != nil { + common.Debug("Unmanaged target directory error: %v (path: %q)", err, path) + return nil + } + if !pathlib.Exists(path) { + it.path = path + it.resolved = true + return nil + } + identityfile := filepath.Join(path, "identity.yaml") + _, identity, err := ComposeFinalBlueprint([]string{identityfile}, "") + if err != nil { + return nil + } + expected := common.BlueprintHash(blueprint) + actual := common.BlueprintHash(identity) + if actual != expected { + it.protected = true + it.resolved = true + return fmt.Errorf("Existing unmanaged space fingerprint %q does not match requested one %q! Quitting!", actual, expected) + } + it.path = path + it.protected = true + it.resolved = true + return nil +} + +func (it *unmanaged) ValidateBlueprint(blueprint []byte) error { + err := it.resolve(blueprint) + if err != nil { + return err + } + if it.protected { + return nil + } + return it.delegate.ValidateBlueprint(blueprint) +} + +func (it *unmanaged) Record(blueprint []byte) error { + it.resolve(blueprint) + if it.protected { + common.Timeline("holotree unmanaged record prevention") + return nil + } + return it.delegate.Record(blueprint) +} + +func (it *unmanaged) WarrantyVoidedDir(controller, space []byte) string { + return it.delegate.WarrantyVoidedDir(controller, space) +} + +func (it *unmanaged) TargetDir(blueprint, client, tag []byte) (string, error) { + return it.delegate.TargetDir(blueprint, client, tag) +} + +func (it *unmanaged) Restore(blueprint, client, tag []byte) (result string, err error) { + return it.RestoreTo(blueprint, ControllerSpaceName(client, tag), string(client), string(tag), false) +} + +func (it *unmanaged) RestoreTo(blueprint []byte, label, controller, space string, partial bool) (result string, err error) { + it.resolve(blueprint) + if !it.protected { + return it.delegate.RestoreTo(blueprint, label, controller, space, partial) + } + common.Timeline("holotree unmanaged restore prevention") + if len(it.path) > 0 { + return it.path, nil + } + return "", fmt.Errorf("Unmanaged path resolution failed!") +} + +func (it *unmanaged) Open(digest string) (readable io.Reader, closer Closer, err error) { + return it.delegate.Open(digest) +} + +func (it *unmanaged) ExactLocation(key string) string { + return it.delegate.ExactLocation(key) +} + +func (it *unmanaged) Location(key string) string { + return it.delegate.Location(key) +} + +func (it *unmanaged) HasBlueprint(blueprint []byte) bool { + return it.delegate.HasBlueprint(blueprint) +} diff --git a/htfs/virtual.go b/htfs/virtual.go new file mode 100644 index 00000000..64499cd1 --- /dev/null +++ b/htfs/virtual.go @@ -0,0 +1,169 @@ +package htfs + +import ( + "fmt" + "io" + "os" + "path/filepath" + + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/fail" + "github.com/robocorp/rcc/journal" + "github.com/robocorp/rcc/pathlib" + "github.com/robocorp/rcc/pretty" +) + +type virtual struct { + identity uint64 + root *Root + registry map[string]string + key string +} + +func Virtual() MutableLibrary { + return &virtual{ + identity: common.Sipit([]byte(common.Product.Home())), + } +} + +func (it *virtual) Compress() bool { + return true +} + +func (it *virtual) Identity() string { + suffix := fmt.Sprintf("%016x", it.identity) + return fmt.Sprintf("v%s_%sh", common.UserHomeIdentity(), suffix[:14]) +} + +func (it *virtual) Stage() string { + stage := filepath.Join(common.HolotreeLocation(), it.Identity()) + err := os.MkdirAll(stage, 0o755) + if err != nil { + panic(err) + } + return stage +} + +func (it *virtual) CatalogPath(key string) string { + return "Virtual Does Not Support Catalog Path Request" +} + +func (it *virtual) Remove([]string) error { + return fmt.Errorf("Not supported yet on virtual holotree.") +} + +func (it *virtual) Export([]string, []string, string) error { + return fmt.Errorf("Not supported yet on virtual holotree.") +} + +func (it *virtual) WriteIdentity([]byte) error { + return fmt.Errorf("Not supported yet on virtual holotree.") +} + +func (it *virtual) Record(blueprint []byte) (err error) { + defer fail.Around(&err) + defer common.Stopwatch("Holotree recording took:").Debug() + key := common.BlueprintHash(blueprint) + common.Timeline("holotree record start %s (virtual)", key) + fs, err := NewRoot(it.Stage()) + fail.On(err != nil, "Failed to create stage root: %v", err) + err = fs.Lift() + fail.On(err != nil, "Failed to lift structure out of stage: %v", err) + common.Timeline("holotree (re)locator start (virtual)") + err = fs.AllFiles(Locator(it.Identity())) + fail.On(err != nil, "Failed to apply relocate to stage: %v", err) + common.Timeline("holotree (re)locator done (virtual)") + it.registry = make(map[string]string) + fs.Treetop(DigestMapper(it.registry)) + fs.Blueprint = key + it.root = fs + it.key = key + return nil +} + +func (it *virtual) WarrantyVoidedDir(controller, space []byte) string { + pretty.Exit(13, "hololib.zip does not support `--warranty-voided` running") + return "" +} + +func (it *virtual) TargetDir(blueprint, client, tag []byte) (string, error) { + name := ControllerSpaceName(client, tag) + return filepath.Join(common.HolotreeLocation(), name), nil +} + +func (it *virtual) Restore(blueprint, client, tag []byte) (result string, err error) { + return it.RestoreTo(blueprint, ControllerSpaceName(client, tag), string(client), string(tag), false) +} + +func (it *virtual) RestoreTo(blueprint []byte, label, controller, space string, partial bool) (result string, err error) { + defer common.Stopwatch("Holotree restore took:").Debug() + key := common.BlueprintHash(blueprint) + common.Timeline("holotree restore start %s (virtual)", key) + targetdir := filepath.Join(common.HolotreeLocation(), label) + metafile := fmt.Sprintf("%s.meta", targetdir) + lockfile := fmt.Sprintf("%s.lck", targetdir) + completed := pathlib.LockWaitMessage(lockfile, "Serialized holotree restore [holotree virtual lock]") + locker, err := pathlib.Locker(lockfile, 30000, common.SharedHolotree) + completed() + fail.On(err != nil, "Could not get lock for %s. Quiting.", targetdir) + defer locker.Release() + journal.Post("space-used", metafile, "virutal holotree with blueprint %s", key) + currentstate := make(map[string]string) + shadow, err := NewRoot(targetdir) + if err == nil { + err = shadow.LoadFrom(metafile) + } + if err == nil { + common.Timeline("holotree digest start (virtual)") + shadow.Treetop(DigestRecorder(currentstate)) + common.Timeline("holotree digest done (virtual)") + } + fs := it.root + err = fs.Relocate(targetdir) + if err != nil { + return "", err + } + common.Timeline("holotree make branches start (virtual)") + err = fs.Treetop(MakeBranches) + common.Timeline("holotree make branches done (virtual)") + if err != nil { + return "", err + } + score := &stats{} + common.Timeline("holotree restore start (virtual)") + err = fs.AllDirs(RestoreDirectory(it, fs, currentstate, score)) + if err != nil { + return "", err + } + common.Timeline("holotree restore done (virtual)") + defer common.Timeline("- dirty %d/%d", score.dirty, score.total) + common.Debug("Holotree dirty workload: %d/%d\n", score.dirty, score.total) + journal.CurrentBuildEvent().Dirty(score.Dirtyness()) + fs.Controller = controller + fs.Space = space + err = fs.SaveAs(metafile) + if err != nil { + return "", err + } + return targetdir, nil +} + +func (it *virtual) Open(digest string) (readable io.Reader, closer Closer, err error) { + return delegateOpen(it, digest, false) +} + +func (it *virtual) ExactLocation(key string) string { + return it.registry[key] +} + +func (it *virtual) Location(key string) string { + panic("Location is not supported on virtual holotree.") +} + +func (it *virtual) ValidateBlueprint(blueprint []byte) error { + return nil +} + +func (it *virtual) HasBlueprint(blueprint []byte) bool { + return it.key == common.BlueprintHash(blueprint) +} diff --git a/htfs/ziplibrary.go b/htfs/ziplibrary.go new file mode 100644 index 00000000..fa56ec0b --- /dev/null +++ b/htfs/ziplibrary.go @@ -0,0 +1,157 @@ +package htfs + +import ( + "archive/zip" + "compress/gzip" + "fmt" + "io" + "path/filepath" + "runtime" + "strings" + + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/fail" + "github.com/robocorp/rcc/journal" + "github.com/robocorp/rcc/pathlib" + "github.com/robocorp/rcc/pretty" +) + +type ziplibrary struct { + content *zip.ReadCloser + identity uint64 + root *Root + lookup map[string]*zip.File +} + +func ZipLibrary(zipfile string) (Library, error) { + content, err := zip.OpenReader(zipfile) + if err != nil { + return nil, err + } + lookup := make(map[string]*zip.File) + for _, entry := range content.File { + lookup[entry.Name] = entry + } + identity := strings.ToLower(fmt.Sprintf("%s %s", runtime.GOOS, runtime.GOARCH)) + return &ziplibrary{ + content: content, + identity: common.Sipit([]byte(identity)), + lookup: lookup, + }, nil +} + +func (it *ziplibrary) ValidateBlueprint(blueprint []byte) error { + return nil +} + +func (it *ziplibrary) HasBlueprint(blueprint []byte) bool { + key := common.BlueprintHash(blueprint) + _, ok := it.lookup[it.CatalogPath(key)] + return ok +} + +func (it *ziplibrary) openFile(filename string) (readable io.Reader, closer Closer, err error) { + content, ok := it.lookup[filename] + if !ok { + return nil, nil, fmt.Errorf("Missing file: %q", filename) + } + file, err := content.Open() + if err != nil { + return nil, nil, err + } + wrapper, err := gzip.NewReader(file) + if err != nil { + return nil, nil, err + } + closer = func() error { + wrapper.Close() + return file.Close() + } + return wrapper, closer, nil +} + +func (it *ziplibrary) Open(digest string) (readable io.Reader, closer Closer, err error) { + filename := filepath.Join("library", digest[:2], digest[2:4], digest[4:6], digest) + return it.openFile(filename) +} + +func (it *ziplibrary) CatalogPath(key string) string { + return filepath.Join("catalog", CatalogName(key)) +} + +func (it *ziplibrary) WarrantyVoidedDir(controller, space []byte) string { + pretty.Exit(13, "hololib.zip does not support `--warranty-voided` running") + return "" +} + +func (it *ziplibrary) TargetDir(blueprint, client, tag []byte) (path string, err error) { + defer fail.Around(&err) + key := common.BlueprintHash(blueprint) + name := ControllerSpaceName(client, tag) + fs, err := NewRoot(".") + fail.On(err != nil, "Failed to create root -> %v", err) + catalog := it.CatalogPath(key) + reader, closer, err := it.openFile(catalog) + fail.On(err != nil, "Failed to open catalog %q -> %v", catalog, err) + defer closer() + err = fs.ReadFrom(reader) + fail.On(err != nil, "Failed to read catalog %q -> %v", catalog, err) + return filepath.Join(fs.HolotreeBase(), name), nil +} + +func (it *ziplibrary) Restore(blueprint, client, tag []byte) (result string, err error) { + return it.RestoreTo(blueprint, ControllerSpaceName(client, tag), string(client), string(tag), false) +} + +func (it *ziplibrary) RestoreTo(blueprint []byte, label, controller, space string, partial bool) (result string, err error) { + defer fail.Around(&err) + defer common.Stopwatch("Holotree restore took:").Debug() + key := common.BlueprintHash(blueprint) + common.Timeline("holotree restore start %s (zip)", key) + fs, err := NewRoot(".") + fail.On(err != nil, "Failed to create root -> %v", err) + catalog := it.CatalogPath(key) + reader, closer, err := it.openFile(catalog) + fail.On(err != nil, "Failed to open catalog %q -> %v", catalog, err) + defer closer() + err = fs.ReadFrom(reader) + fail.On(err != nil, "Failed to read catalog %q -> %v", catalog, err) + targetdir := filepath.Join(fs.HolotreeBase(), label) + metafile := fmt.Sprintf("%s.meta", targetdir) + lockfile := fmt.Sprintf("%s.lck", targetdir) + completed := pathlib.LockWaitMessage(lockfile, "Serialized holotree restore [holotree base lock]") + locker, err := pathlib.Locker(lockfile, 30000, common.SharedHolotree) + completed() + fail.On(err != nil, "Could not get lock for %s. Quiting.", targetdir) + defer locker.Release() + journal.Post("space-used", metafile, "zipped holotree with blueprint %s from %s", key, catalog) + currentstate := make(map[string]string) + shadow, err := NewRoot(targetdir) + if err == nil { + err = shadow.LoadFrom(metafile) + } + if err == nil { + common.TimelineBegin("holotree digest start (zip)") + shadow.Treetop(DigestRecorder(currentstate)) + common.TimelineEnd() + } + err = fs.Relocate(targetdir) + fail.On(err != nil, "Failed to relocate %q -> %v", targetdir, err) + common.TimelineBegin("holotree make branches start (zip)") + err = fs.Treetop(MakeBranches) + common.TimelineEnd() + fail.On(err != nil, "Failed to make branches %q -> %v", targetdir, err) + score := &stats{} + common.TimelineBegin("holotree restore start (zip)") + err = fs.AllDirs(RestoreDirectory(it, fs, currentstate, score)) + fail.On(err != nil, "Failed to restore directory %q -> %v", targetdir, err) + common.TimelineEnd() + defer common.Timeline("- dirty %d/%d", score.dirty, score.total) + common.Debug("Holotree dirty workload: %d/%d\n", score.dirty, score.total) + journal.CurrentBuildEvent().Dirty(score.Dirtyness()) + fs.Controller = controller + fs.Space = space + err = fs.SaveAs(metafile) + fail.On(err != nil, "Failed to save metafile %q -> %v", metafile, err) + return targetdir, nil +} diff --git a/journal/buildstats.go b/journal/buildstats.go new file mode 100644 index 00000000..6c30228c --- /dev/null +++ b/journal/buildstats.go @@ -0,0 +1,598 @@ +package journal + +import ( + "bufio" + "bytes" + "encoding/json" + "fmt" + "io" + "math" + "os" + "path/filepath" + "sort" + "strings" + "text/tabwriter" + "time" + + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/fail" + "github.com/robocorp/rcc/pathlib" + "github.com/robocorp/rcc/pretty" +) + +type ( + acceptor func(*BuildEvent) bool + picker func(*BuildEvent) float64 + flagger func(*BuildEvent) bool + prettify func(float64) string + + Numbers []float64 + BuildEvents []*BuildEvent + BuildEvent struct { + Version string `json:"version"` + When int64 `json:"when"` + What string `json:"what"` + Force bool `json:"force"` + Build bool `json:"build"` + Success bool `json:"success"` + Retry bool `json:"retry"` + Run bool `json:"run"` + Controller string `json:"controller"` + Space string `json:"space"` + BlueprintHash string `json:"blueprint"` + + Started float64 `json:"started"` + Prepared float64 `json:"prepared"` + MicromambaDone float64 `json:"micromamba"` + PipDone float64 `json:"pip"` + PostInstallDone float64 `json:"postinstall"` + RecordDone float64 `json:"record"` + RestoreDone float64 `json:"restore"` + PreRunDone float64 `json:"prerun"` + RobotStart float64 `json:"robotstart"` + RobotEnd float64 `json:"robotend"` + Finished float64 `json:"finished"` + Dirtyness float64 `json:"dirtyness"` + } +) + +const ( + assistantKey = "assistant" + prepareKey = "prepare" + robotKey = "robot" + variableKey = "variables" +) + +var ( + buildevent *BuildEvent +) + +func init() { + buildevent = NewBuildEvent() +} + +func CurrentBuildEvent() *BuildEvent { + return buildevent +} + +func BuildEventStats(label string) { + err := serialize(buildevent.finished(label)) + if err != nil { + pretty.Warning("build stats for %q failed, reason: %v", label, err) + } +} + +func asPercent(value float64) string { + return fmt.Sprintf("%5.1f%%", value) +} + +func asSecond(value float64) string { + return fmt.Sprintf("%7.3fs", value) +} + +func asCount(value int) string { + return fmt.Sprintf("%d", value) +} + +func forced(the *BuildEvent) bool { + return the.Force +} + +func retried(the *BuildEvent) bool { + return the.Retry +} + +func failed(the *BuildEvent) bool { + return the.Build && !the.Success +} + +func build(the *BuildEvent) bool { + return the.Build +} + +func started(the *BuildEvent) float64 { + return the.Started +} + +func prepared(the *BuildEvent) float64 { + return the.Prepared +} + +func micromamba(the *BuildEvent) float64 { + return the.MicromambaDone +} + +func pip(the *BuildEvent) float64 { + return the.PipDone +} + +func postinstall(the *BuildEvent) float64 { + return the.PostInstallDone +} + +func record(the *BuildEvent) float64 { + return the.RecordDone +} + +func restore(the *BuildEvent) float64 { + return the.RestoreDone +} + +func prerun(the *BuildEvent) float64 { + return the.PreRunDone +} + +func robotstarts(the *BuildEvent) float64 { + return the.RobotStart +} + +func robotends(the *BuildEvent) float64 { + return the.RobotEnd +} + +func finished(the *BuildEvent) float64 { + return the.Finished +} + +func anyOf(flags ...bool) bool { + for _, flag := range flags { + if flag { + return true + } + } + return false +} + +func ShowStatistics(weeks uint, assistants, robots, prepares, variables bool) { + _, body := MakeStatistics(weeks, assistants, robots, prepares, variables) + os.Stderr.Write(body) + os.Stderr.Sync() +} + +func MakeStatistics(weeks uint, assistants, robots, prepares, variables bool) (int, []byte) { + sink := bytes.NewBuffer(nil) + stats, err := Stats(weeks) + selected := []string{"all"} + if err != nil { + pretty.Warning("Loading statistics failed, reason: %v", err) + return 0, sink.Bytes() + } + if anyOf(assistants, robots, prepares, variables) { + selectors := make(map[string]bool) + selectors[assistantKey] = assistants + selectors[robotKey] = robots + selectors[prepareKey] = prepares + selectors[variableKey] = variables + stats = stats.filter(func(the *BuildEvent) bool { + return selectors[the.What] + }) + selected = []string{} + for key, value := range selectors { + if value { + selected = append(selected, key) + } + } + sort.Strings(selected) + } + tabbed := tabwriter.NewWriter(sink, 2, 4, 2, ' ', tabwriter.AlignRight) + tabbed.Write(sprint("Selected (%s) statistics: %d samples [%d full weeks]\t\n", strings.Join(selected, ", "), len(stats), weeks)) + tabbed.Write([]byte("\n")) + tabbed.Write([]byte("Name \tAverage\t10%\tMedian\t90%\tMAX\t\n")) + stats.Statsline(tabbed, "Dirty", asPercent, func(the *BuildEvent) float64 { + return the.Dirtyness + }) + tabbed.Write([]byte("\n")) + tabbed.Write([]byte("Name \tAverage\t10%\tMedian\t90%\tMAX\t\n")) + stats.Statsline(tabbed, "Lead time ", asSecond, func(the *BuildEvent) float64 { + return the.Started + }) + stats.Statsline(tabbed, "Setup time ", asSecond, func(the *BuildEvent) float64 { + if the.RobotStart > 0 { + return the.RobotStart - the.Started + } + return the.Finished - the.Started + }) + stats.Statsline(tabbed, "Holospace restore time", asSecond, func(the *BuildEvent) float64 { + if the.RestoreDone > 0 { + return the.RestoreDone - the.Started + } + return the.Finished - the.Started + }) + stats.Statsline(tabbed, "Pre-run ", asSecond, func(the *BuildEvent) float64 { + if the.PreRunDone > 0 { + return the.PreRunDone - the.RestoreDone + } + return 0 + }) + stats.Statsline(tabbed, "Robot startup delay ", asSecond, func(the *BuildEvent) float64 { + return the.RobotStart + }) + stats.Statsline(tabbed, "Robot execution time ", asSecond, func(the *BuildEvent) float64 { + return the.RobotEnd - the.RobotStart + }) + stats.Statsline(tabbed, "Total execution time ", asSecond, func(the *BuildEvent) float64 { + return the.Finished + }) + onlyBuilds := stats.filter(func(the *BuildEvent) bool { + return the.Build + }) + tabbed.Write([]byte("\n\n")) + statCount := len(stats) + percentage := 100.0 * float64(len(onlyBuilds)) / float64(statCount) + tabbed.Write(sprint("%d\tsamples with environment builds\t(%3.1f%% from selected)\t\n", len(onlyBuilds), percentage)) + tabbed.Write([]byte("\n")) + tabbed.Write([]byte("Name \tAverage\t10%\tMedian\t90%\tMAX\t\n")) + onlyBuilds.Statsline(tabbed, "Phase: prepare ", asSecond, func(the *BuildEvent) float64 { + if the.Prepared > 0 { + return the.Prepared - the.Started + } + return 0.0 + }) + onlyBuilds.Statsline(tabbed, "Phase: micromamba ", asSecond, func(the *BuildEvent) float64 { + if the.MicromambaDone > 0 { + return the.MicromambaDone - the.Prepared + } + return 0.0 + }) + onlyBuilds.Statsline(tabbed, "Phase: pip ", asSecond, func(the *BuildEvent) float64 { + if the.PipDone > 0 { + return the.PipDone - the.MicromambaDone + } + return 0.0 + }) + onlyBuilds.Statsline(tabbed, "Phase: post install ", asSecond, func(the *BuildEvent) float64 { + if the.PostInstallDone > 0 { + return the.PostInstallDone - the.first(pip, micromamba) + } + return 0.0 + }) + onlyBuilds.Statsline(tabbed, "Phase: record ", asSecond, func(the *BuildEvent) float64 { + if the.RecordDone > 0 { + return the.RecordDone - the.first(postinstall, pip, micromamba) + } + return 0.0 + }) + onlyBuilds.Statsline(tabbed, "To hololib ", asSecond, func(the *BuildEvent) float64 { + if the.RecordDone > 0 { + return the.RecordDone - the.Started + } + return 0.0 + }) + + assistantStats := selectStats(stats, assistantKey) + prepareStats := selectStats(stats, prepareKey) + robotStats := selectStats(stats, robotKey) + variableStats := selectStats(stats, variableKey) + + tabbed.Write([]byte("\n\n")) + tabbed.Write([]byte("Cumulative \tAssistants\tPrepares\tRobots\tVariables\tTotal\t\n")) + tabbed.Write(tabs("Run counts ", theSize(assistantStats), theSize(prepareStats), theSize(robotStats), theSize(variableStats), theSize(stats))) + tabbed.Write(tabs("Build counts ", + theCounts(assistantStats, build), + theCounts(prepareStats, build), + theCounts(robotStats, build), + theCounts(variableStats, build), + theCounts(stats, build))) + tabbed.Write(tabs("Forced builds ", + theCounts(assistantStats, forced), + theCounts(prepareStats, forced), + theCounts(robotStats, forced), + theCounts(variableStats, forced), + theCounts(stats, forced))) + tabbed.Write(tabs("Retried builds", + theCounts(assistantStats, retried), + theCounts(prepareStats, retried), + theCounts(robotStats, retried), + theCounts(variableStats, retried), + theCounts(stats, retried))) + tabbed.Write(tabs("Failed builds ", + theCounts(assistantStats, failed), + theCounts(prepareStats, failed), + theCounts(robotStats, failed), + theCounts(variableStats, failed), + theCounts(stats, failed))) + tabbed.Write(tabs("Setup times ", + theTimes(assistantStats, priority(robotstarts, finished), priority(started)), + theTimes(prepareStats, priority(robotstarts, finished), priority(started)), + theTimes(robotStats, priority(robotstarts, finished), priority(started)), + theTimes(variableStats, priority(robotstarts, finished), priority(started)), + theTimes(stats, priority(robotstarts, finished), priority(started)))) + tabbed.Write(tabs("Run times ", + theTimes(assistantStats, priority(robotends), priority(robotstarts)), + theTimes(prepareStats, priority(robotends), priority(robotstarts)), + theTimes(robotStats, priority(robotends), priority(robotstarts)), + theTimes(variableStats, priority(robotends), priority(robotstarts)), + theTimes(stats, priority(robotends), priority(robotstarts)))) + tabbed.Write(tabs("Total times ", + theTimes(assistantStats, priority(finished), priority(started)), + theTimes(prepareStats, priority(finished), priority(started)), + theTimes(robotStats, priority(finished), priority(started)), + theTimes(variableStats, priority(finished), priority(started)), + theTimes(stats, priority(finished), priority(started)))) + tabbed.Flush() + return statCount, sink.Bytes() +} + +func theCounts(source BuildEvents, check flagger) string { + total := 0 + for _, event := range source { + if check(event) { + total++ + } + } + return asCount(total) +} + +func priority(pickers ...picker) []picker { + return pickers +} + +func theTimes(source BuildEvents, till []picker, from []picker) string { + total := 0.0 + for _, event := range source { + done := event.first(till...) + if done > 0.0 { + area := done - event.first(from...) + total += area + } + } + return asSecond(total) +} + +func theSize(source BuildEvents) string { + return fmt.Sprintf("%d", len(source)) +} + +func selectStats(source BuildEvents, key string) BuildEvents { + result := source.filter(func(the *BuildEvent) bool { + return the.What == key + }) + return result +} + +func tabs(columns ...any) []byte { + form := strings.Repeat("%s\t", len(columns)) + "\n" + return []byte(fmt.Sprintf(form, columns...)) +} + +func BuildEventFilenameFor(stamp time.Time) string { + year, week := stamp.ISOWeek() + filename := fmt.Sprintf("stats_%s_%04d_%02d.log", common.UserHomeIdentity(), year, week) + return filepath.Join(common.JournalLocation(), filename) +} + +func CurrentEventFilename() string { + return BuildEventFilenameFor(common.Clock.Time()) +} + +func BuildEventFilenamesFor(weekcount int) []string { + weekstep := -7 * 24 * time.Hour + timestamp := common.Clock.Time() + result := make([]string, 0, weekcount+1) + for weekcount >= 0 { + result = append(result, BuildEventFilenameFor(timestamp)) + timestamp = timestamp.Add(weekstep) + weekcount-- + } + return result +} + +func serialize(event *BuildEvent) (err error) { + defer fail.Around(&err) + + blob, err := json.Marshal(event) + fail.On(err != nil, "Could not serialize event: %v -> %v", event.What, err) + return AppendJournal(CurrentEventFilename(), blob) +} + +func NewBuildEvent() *BuildEvent { + return &BuildEvent{ + When: common.Clock.When(), + Version: common.Version, + } +} + +func (it *BuildEvent) stowatch() float64 { + return common.Clock.Elapsed().Seconds() +} + +func (it *BuildEvent) finished(label string) *BuildEvent { + it.What = label + it.Finished = it.stowatch() + it.Controller = common.ControllerType + it.Space = common.HolotreeSpace + return it +} + +func (it *BuildEvent) Successful() { + it.Success = true +} + +func (it *BuildEvent) StartNow(force bool) { + it.Started = it.stowatch() + it.Force = force +} + +func (it *BuildEvent) Blueprint(blueprint string) { + it.BlueprintHash = blueprint +} + +func (it *BuildEvent) Rebuild() { + it.Retry = true + it.Build = true +} + +func (it *BuildEvent) PrepareComplete() { + it.Build = true + it.Prepared = it.stowatch() +} + +func (it *BuildEvent) MicromambaComplete() { + it.Build = true + it.MicromambaDone = it.stowatch() +} + +func (it *BuildEvent) PipComplete() { + it.Build = true + it.PipDone = it.stowatch() +} + +func (it *BuildEvent) PostInstallComplete() { + it.Build = true + it.PostInstallDone = it.stowatch() +} + +func (it *BuildEvent) RecordComplete() { + it.RecordDone = it.stowatch() +} + +func (it *BuildEvent) Dirty(dirtyness float64) { + it.Dirtyness = dirtyness +} + +func (it *BuildEvent) RestoreComplete() { + it.RestoreDone = it.stowatch() +} + +func (it *BuildEvent) PreRunComplete() { + it.Run = true + it.PreRunDone = it.stowatch() +} + +func (it *BuildEvent) RobotStarts() { + it.Run = true + it.RobotStart = it.stowatch() +} + +func (it *BuildEvent) RobotEnds() { + it.Run = true + it.RobotEnd = it.stowatch() + reportRatio("Build/Pre-run", it.first(prerun, restore)-it.Started, it.first(prerun, restore)-it.RestoreDone) + reportRatio("Setup/Run", it.first(prerun, restore)-it.Started, it.RobotEnd-it.first(prerun, restore)) +} + +func reportRatio(label string, first, second float64) { + left := int64(math.Ceil(10 * first)) + right := int64(math.Ceil(10 * second)) + gcd := common.Gcd(left, right) + pretty.Lowlight(" | %q relative time allocation ratio: %d:%d", label, left/gcd, right/gcd) +} + +func (it *BuildEvent) first(tools ...picker) float64 { + for _, tool := range tools { + value := tool(it) + if value > 0 { + return value + } + } + return 0 +} + +func (it BuildEvents) filter(query acceptor) BuildEvents { + result := make(BuildEvents, 0, len(it)) + for _, event := range it { + if query(event) { + result = append(result, event) + } + } + return result +} + +func (it BuildEvents) pick(tool picker) Numbers { + result := make(Numbers, 0, len(it)) + for _, event := range it { + result = append(result, tool(event)) + } + return result +} + +func sprint(form string, fields ...any) []byte { + return []byte(fmt.Sprintf(form, fields...)) +} + +func (it BuildEvents) Statsline(tabbed *tabwriter.Writer, label string, nice prettify, tool picker) { + numbers := it.pick(tool) + sort.Float64s(numbers) + average, low, median, high, last := numbers.Statsline() + tabbed.Write(sprint("%s\t%s\t%s\t%s\t%s\t%s\t\n", label, nice(average), nice(low), nice(median), nice(high), nice(last))) +} + +func (it Numbers) safe(at int) float64 { + total := len(it) + if total == 0 { + return 0.0 + } + if at < 0 { + return it[0] + } + if at < total { + return it[at] + } + return it[total-1] +} + +func (it Numbers) Statsline() (average, low, median, high, worst float64) { + total := len(it) + if total < 1 { + return + } + sum := 0.0 + for _, value := range it { + sum += value + } + half := total >> 1 + percentile := total / 10 + last := total - 1 + right := last - percentile + return sum / float64(total), it.safe(percentile), it.safe(half), it.safe(right), it.safe(last) +} + +func Stats(weeks uint) (result BuildEvents, err error) { + defer fail.Around(&err) + result = make(BuildEvents, 0, 100) + for _, journalname := range BuildEventFilenamesFor(int(weeks)) { + if !pathlib.IsFile(journalname) { + continue + } + handle, err := os.Open(journalname) + fail.On(err != nil, "Failed to open event journal %v -> %v", journalname, err) + defer handle.Close() + source := bufio.NewReader(handle) + fail.On(err != nil, "Failed to read %s.", journalname) + innerloop: + for { + line, err := source.ReadBytes('\n') + if err == io.EOF { + break innerloop + } + fail.On(err != nil, "Failed to read %s.", journalname) + event := &BuildEvent{} + err = json.Unmarshal(line, event) + if err != nil { + continue innerloop + } + result = append(result, event) + } + } + return result, nil +} diff --git a/journal/journal.go b/journal/journal.go new file mode 100644 index 00000000..929a19cf --- /dev/null +++ b/journal/journal.go @@ -0,0 +1,123 @@ +package journal + +import ( + "bufio" + "encoding/json" + "fmt" + "io" + "os" + "regexp" + "strings" + "sync" + "time" + + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/fail" +) + +var ( + spacePattern = regexp.MustCompile("\\s+") + runJournal *journal +) + +type ( + journal struct { + sync.Mutex + filename string + } + Event struct { + When int64 `json:"when"` + Controller string `json:"controller"` + Event string `json:"event"` + Detail string `json:"detail"` + Comment string `json:"comment,omitempty"` + } +) + +func init() { + runJournal = &journal{sync.Mutex{}, ""} + common.RegisterJournal(runJournal) +} + +func ForRun(filename string) { + runJournal.Lock() + defer runJournal.Unlock() + runJournal.filename = filename +} + +func StopRunJournal() { + ForRun("") +} + +func (it *journal) Post(event, detail, commentForm string, fields ...interface{}) (err error) { + if it == nil || len(it.filename) == 0 { + return nil + } + defer fail.Around(&err) + it.Lock() + defer it.Unlock() + message := Event{ + When: time.Now().Unix(), + Controller: common.ControllerIdentity(), + Event: Unify(event), + Detail: detail, + Comment: Unify(fmt.Sprintf(commentForm, fields...)), + } + blob, err := json.Marshal(message) + fail.On(err != nil, "Could not serialize event: %v -> %v", event, err) + return AppendJournal(it.filename, blob) +} + +func Unify(value string) string { + return strings.TrimSpace(spacePattern.ReplaceAllString(value, " ")) +} + +func Post(event, detail, commentForm string, fields ...interface{}) (err error) { + defer fail.Around(&err) + message := Event{ + When: common.When, + Controller: common.ControllerIdentity(), + Event: Unify(event), + Detail: detail, + Comment: Unify(fmt.Sprintf(commentForm, fields...)), + } + blob, err := json.Marshal(message) + fail.On(err != nil, "Could not serialize event: %v -> %v", event, err) + return AppendJournal(common.EventJournal(), blob) +} + +func AppendJournal(journalname string, blob []byte) (err error) { + defer fail.Around(&err) + if common.WarrantyVoided() { + return nil + } + handle, err := os.OpenFile(journalname, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o640) + fail.On(err != nil, "Failed to open event journal %v -> %v", journalname, err) + defer handle.Close() + _, err = handle.Write(append(blob, '\n')) + fail.On(err != nil, "Failed to write event journal %v -> %v", journalname, err) + return handle.Sync() +} + +func Events() (result []Event, err error) { + defer fail.Around(&err) + handle, err := os.Open(common.EventJournal()) + fail.On(err != nil, "Failed to open event journal %v -> %v", common.EventJournal(), err) + defer handle.Close() + source := bufio.NewReader(handle) + fail.On(err != nil, "Failed to read %s.", common.EventJournal()) + result = make([]Event, 0, 100) + for { + line, err := source.ReadBytes('\n') + if err == io.EOF { + return result, nil + } + fail.On(err != nil, "Failed to read %s.", common.EventJournal()) + event := Event{} + err = json.Unmarshal(line, &event) + if err != nil { + continue + } + result = append(result, event) + } +} diff --git a/journal/journal_test.go b/journal/journal_test.go new file mode 100644 index 00000000..9292e002 --- /dev/null +++ b/journal/journal_test.go @@ -0,0 +1,26 @@ +package journal_test + +import ( + "testing" + + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/hamlet" + "github.com/robocorp/rcc/journal" +) + +func TestJounalCanBeCalled(t *testing.T) { + must, wont := hamlet.Specifications(t) + + must.Equal("foo bar", journal.Unify(" foo \t \r\n bar ")) + + common.ControllerType = "unittest" + + must.Nil(journal.Post("unittest", "journal-1", "from journal/journal_test.go")) + events, err := journal.Events() + must.Nil(err) + wont.Nil(events) + must.True(len(events) > 0) + must.Nil(journal.Post("unittest", "journal-2", "from journal/journal_test.go")) + second, err := journal.Events() + must.True(len(second) > len(events)) +} diff --git a/mocks/client.go b/mocks/client.go index 8916340d..a07858d3 100644 --- a/mocks/client.go +++ b/mocks/client.go @@ -2,6 +2,7 @@ package mocks import ( "testing" + "time" "github.com/robocorp/rcc/cloud" ) @@ -28,6 +29,18 @@ func (it *MockClient) NewClient(endpoint string) (cloud.Client, error) { return it, nil } +func (it *MockClient) Uncritical() cloud.Client { + return it +} + +func (it *MockClient) WithTimeout(time.Duration) cloud.Client { + return it +} + +func (it *MockClient) WithTracing() cloud.Client { + return it +} + func (it *MockClient) NewRequest(url string) *cloud.Request { return &cloud.Request{ Url: url, @@ -48,6 +61,10 @@ func (it *MockClient) does(method string, request *cloud.Request) *cloud.Respons return it.Responses[index] } +func (it *MockClient) Head(request *cloud.Request) *cloud.Response { + return it.does("HEAD", request) +} + func (it *MockClient) Get(request *cloud.Request) *cloud.Response { return it.does("GET", request) } diff --git a/operations/assistant.go b/operations/assistant.go index 225bcaee..171ad9ac 100644 --- a/operations/assistant.go +++ b/operations/assistant.go @@ -6,7 +6,6 @@ import ( "errors" "fmt" "io" - "io/ioutil" "mime/multipart" "net/http" "net/url" @@ -18,6 +17,7 @@ import ( "github.com/robocorp/rcc/cloud" "github.com/robocorp/rcc/common" "github.com/robocorp/rcc/pathlib" + "github.com/robocorp/rcc/settings" ) const ( @@ -99,7 +99,7 @@ func (it *ArtifactPublisher) Publish(fullpath, relativepath string, details os.F size, ok := pathlib.Size(fullpath) if !ok { it.ErrorCount += 1 - return //errors.New(fmt.Sprintf("Could not publish file %v, reason: could not determine size!", fullpath)) + return //fmt.Errorf("Could not publish file %v, reason: could not determine size!", fullpath) } client, url, err := it.NewClient(it.ArtifactPostURL) if err != nil { @@ -127,6 +127,7 @@ func (it *ArtifactPublisher) Publish(fullpath, relativepath string, details os.F return //err } if response.Status < 200 || 299 < response.Status { + it.ErrorCount += 1 common.Log("ERR: status code %v", response.Status) return //err } @@ -147,14 +148,14 @@ func (it *ArtifactPublisher) Publish(fullpath, relativepath string, details os.F common.Log("ERR: did not get correct response postinfo in reply from cloud.") return //err } - err = multipartUpload(outcome.Response.PostInfo.Url, outcome.Response.PostInfo.Fields, basename, fullpath) + err = MultipartUpload(outcome.Response.PostInfo.Url, outcome.Response.PostInfo.Fields, basename, fullpath) if err != nil { it.ErrorCount += 1 common.Error("Assistant/Last", err) } } -func multipartUpload(url string, fields map[string]string, basename, fullpath string) error { +func MultipartUpload(url string, fields map[string]string, basename, fullpath string) error { buffer := new(bytes.Buffer) many := multipart.NewWriter(buffer) @@ -187,38 +188,26 @@ func multipartUpload(url string, fields map[string]string, basename, fullpath st return err } request.Header.Add("Content-Type", many.FormDataContentType()) - client := &http.Client{} + request.Header.Add("User-Agent", common.UserAgent()) + client := &http.Client{Transport: settings.Global.ConfiguredHttpTransport()} response, err := client.Do(request) if err != nil { return err } if response.StatusCode < 200 || response.StatusCode > 299 { - return errors.New(fmt.Sprintf("Warning: status: %d reason: %s", response.StatusCode, IoAsString(response.Body))) + return fmt.Errorf("Warning: status: %d reason: %s", response.StatusCode, IoAsString(response.Body)) } return nil } func IoAsString(source io.Reader) string { - body, err := ioutil.ReadAll(source) + body, err := io.ReadAll(source) if err != nil { return "" } return string(body) } -func AssistantTreeCommand(client cloud.Client, account *account, workspace string) (*WorkspaceTreeData, error) { - response, err := WorkspaceTreeCommandRequest(client, account, workspace) - if err != nil { - return nil, err - } - treedata := new(WorkspaceTreeData) - err = json.Unmarshal(response.Body, &treedata) - if err != nil { - return nil, err - } - return treedata, nil -} - func ListAssistantsCommand(client cloud.Client, account *account, workspaceId string) ([]Token, error) { credentials, err := summonAssistantToken(client, account, workspaceId) if err != nil { @@ -228,7 +217,7 @@ func ListAssistantsCommand(client cloud.Client, account *account, workspaceId st request.Headers[authorization] = WorkspaceToken(credentials) response := client.Get(request) if response.Status != 200 { - return nil, errors.New(fmt.Sprintf("%d: %s", response.Status, response.Body)) + return nil, fmt.Errorf("%d: %s", response.Status, response.Body) } tokens := make([]Token, 100) err = json.Unmarshal(response.Body, &tokens) @@ -245,15 +234,23 @@ func BackgroundAssistantHeartbeat(cancel chan bool, client cloud.Client, account case _ = <-cancel: common.Trace("Stopping assistant heartbeat.") return - case <-time.After(60 * time.Second): + case <-time.After(37 * time.Second): counter += 1 common.Trace("Sending assistant heartbeat #%d.", counter) - go BeatAssistantRun(client, account, workspaceId, assistantId, runId, counter) + go beatAssistantRunBackground(client, account, workspaceId, assistantId, runId, counter) } } } +func beatAssistantRunBackground(client cloud.Client, account *account, workspaceId, assistantId, runId string, beat int) { + err := BeatAssistantRun(client, account, workspaceId, assistantId, runId, beat) + if err != nil { + common.Log("Problem sendig assistant heartbeat: %s", err) + } +} + func BeatAssistantRun(client cloud.Client, account *account, workspaceId, assistantId, runId string, beat int) error { + common.Timeline("send assistant heartbeat") credentials, err := summonAssistantToken(client, account, workspaceId) if err != nil { return err @@ -262,18 +259,20 @@ func BeatAssistantRun(client cloud.Client, account *account, workspaceId, assist token["seq"] = beat request := client.NewRequest(fmt.Sprintf(beatAssistantApi, workspaceId, assistantId, runId)) request.Headers[authorization] = WorkspaceToken(credentials) + request.Headers[contentType] = applicationJson blob, err := json.Marshal(token) if err == nil { request.Body = bytes.NewReader(blob) } response := client.Post(request) if response.Status != 200 { - return errors.New(fmt.Sprintf("%d: %s", response.Status, response.Body)) + return fmt.Errorf("%d: %s", response.Status, response.Body) } return nil } func StopAssistantRun(client cloud.Client, account *account, workspaceId, assistantId, runId, status, reason string) error { + common.Timeline("stop assistant run: %s", assistantId) credentials, err := summonAssistantToken(client, account, workspaceId) if err != nil { return err @@ -283,18 +282,20 @@ func StopAssistantRun(client cloud.Client, account *account, workspaceId, assist token["error"] = reason request := client.NewRequest(fmt.Sprintf(stopAssistantApi, workspaceId, assistantId, runId)) request.Headers[authorization] = WorkspaceToken(credentials) + request.Headers[contentType] = applicationJson blob, err := json.Marshal(token) if err == nil { request.Body = bytes.NewReader(blob) } response := client.Put(request) if response.Status != 200 { - return errors.New(fmt.Sprintf("%d: %s", response.Status, response.Body)) + return fmt.Errorf("%d: %s", response.Status, response.Body) } return nil } func StartAssistantRun(client cloud.Client, account *account, workspaceId, assistantId string) (*AssistantRobot, error) { + common.Timeline("start assistant run: %q", assistantId) credentials, err := summonAssistantToken(client, account, workspaceId) if err != nil { return nil, err @@ -303,15 +304,17 @@ func StartAssistantRun(client cloud.Client, account *account, workspaceId, assis if err != nil { return nil, err } + common.Timeline("start assistant run CR network request") request := client.NewRequest(fmt.Sprintf(startAssistantApi, workspaceId, assistantId)) request.Headers[authorization] = WorkspaceToken(credentials) + request.Headers[contentType] = applicationJson request.Body, err = key.RequestBody(nil) if err != nil { return nil, err } response := client.Post(request) if response.Status != 200 { - return nil, errors.New(fmt.Sprintf("%d: %s", response.Status, response.Body)) + return nil, fmt.Errorf("%d: %s", response.Status, response.Body) } plaintext, err := key.Decode(response.Body) if err != nil { diff --git a/operations/authorize.go b/operations/authorize.go index fac21e57..848652f1 100644 --- a/operations/authorize.go +++ b/operations/authorize.go @@ -5,12 +5,12 @@ import ( "crypto/sha256" "encoding/base64" "encoding/json" - "errors" "fmt" "strings" "time" "github.com/robocorp/rcc/cloud" + "github.com/robocorp/rcc/common" ) const ( @@ -30,15 +30,12 @@ const ( newline = "\n" ) -type Capability map[string]bool -type Capabilities map[string]Capability - type Claims struct { - ExpiresIn int `json:"expiresIn,omitempty"` - Capabilities Capabilities `json:"capabilities,omitempty"` - Method string `json:"-"` - Url string `json:"-"` - Name string `json:"-"` + ExpiresIn int `json:"expiresIn,omitempty"` + CapabilitySet string `json:"capabilitySet,omitempty"` + Method string `json:"-"` + Url string `json:"-"` + Name string `json:"-"` } type Token map[string]interface{} @@ -51,6 +48,14 @@ func (it Token) AsJson() (string, error) { return string(body), nil } +func (it Token) FromJson(content []byte) error { + err := json.Unmarshal(content, &it) + if err != nil { + return err + } + return nil +} + type UserInfo struct { User Token `json:"user"` Link Token `json:"request"` @@ -58,11 +63,10 @@ type UserInfo struct { func NewClaims(name, url string, expires int) *Claims { result := Claims{ - ExpiresIn: expires, - Capabilities: make(Capabilities), - Url: url, - Name: name, - Method: postMethod, + ExpiresIn: expires, + Url: url, + Name: name, + Method: postMethod, } return &result } @@ -89,45 +93,34 @@ func (it *Claims) AsJson() (string, error) { return string(body), nil } -func (it Capabilities) Add(name string, list, read, write bool) { - capability := make(Capability) - capability["list"] = list - capability["read"] = read - capability["write"] = write - it[name] = capability -} - -func ActivityClaims(seconds int, workspace string) *Claims { - result := NewClaims("Activity", fmt.Sprintf(WorkspaceApi, workspace), seconds) - result.Capabilities.Add("activity", true, true, true) +func EditRobotClaims(seconds int, workspace string) *Claims { + result := NewClaims("EditRobot", fmt.Sprintf(WorkspaceApi, workspace), seconds) + result.CapabilitySet = "edit/robot" return result } -func AssistantClaims(seconds int, workspace string) *Claims { - result := NewClaims("Assistant", fmt.Sprintf(WorkspaceApi, workspace), seconds) - result.Capabilities.Add("assistant", true, true, true) +func RunAssistantClaims(seconds int, workspace string) *Claims { + result := NewClaims("RunAssistant", fmt.Sprintf(WorkspaceApi, workspace), seconds) + result.CapabilitySet = "run/assistant" return result } -func RobotClaims(seconds int, workspace string) *Claims { - result := NewClaims("Robot", fmt.Sprintf(WorkspaceApi, workspace), seconds) - result.Capabilities.Add("package", true, true, true) +func RunRobotClaims(seconds int, workspace string) *Claims { + result := NewClaims("RunRobot", fmt.Sprintf(WorkspaceApi, workspace), seconds) + result.CapabilitySet = "run/robot" + cloud.InternalBackgroundMetric(common.ControllerIdentity(), "rcc.capabilityset.runrobot", common.Version) return result } -func RunClaims(seconds int, workspace string) *Claims { - result := NewClaims("Run", fmt.Sprintf(WorkspaceApi, workspace), seconds) - result.Capabilities.Add("secret", true, true, false) - result.Capabilities.Add("artifact", false, false, true) - result.Capabilities.Add("livedata", false, true, true) - result.Capabilities.Add("workitemdata", false, true, true) +func GetRobotClaims(seconds int, workspace string) *Claims { + result := NewClaims("GetRobot", fmt.Sprintf(WorkspaceApi, workspace), seconds) + result.CapabilitySet = "get/robot" return result } -func WorkspaceTreeClaims(seconds int) *Claims { - result := NewClaims("User", UserApi, seconds) - result.Capabilities.Add("workspace", true, false, false) - result.Capabilities.Add("workspaceTree", true, true, false) +func ViewWorkspacesClaims(seconds int) *Claims { + result := NewClaims("ViewWorkspaces", UserApi, seconds) + result.CapabilitySet = "view/workspaces" return result } @@ -147,7 +140,7 @@ func WorkspaceToken(token string) string { return fmt.Sprintf("RC_WST %s", token) } -func RobocorpCloudHmac(identifier, token string) string { +func ProductCloudHmac(identifier, token string) string { return fmt.Sprintf("robocloud-hmac %s %s", identifier, token) } @@ -164,25 +157,25 @@ func HmacSignature(claims *Claims, secret, nonce, bodyHash string) string { return base64.StdEncoding.EncodeToString(hasher.Sum(nil)) } -func AuthorizeClaims(accountName string, claims *Claims) (Token, error) { +func AuthorizeClaims(accountName string, claims *Claims, period *TokenPeriod) (Token, error) { account := AccountByName(accountName) if account == nil { - return nil, errors.New(fmt.Sprintf("Could not find account by name: %s", accountName)) + return nil, fmt.Errorf("Could not find account by name: %q", accountName) } client, err := cloud.NewClient(account.Endpoint) if err != nil { - return nil, errors.New(fmt.Sprintf("Could not create client for endpoint: %s reason: %s", account.Endpoint, err)) + return nil, fmt.Errorf("Could not create client for endpoint: %s reason: %w", account.Endpoint, err) } - data, err := AuthorizeCommand(client, account, claims) + data, err := AuthorizeCommand(client, account, claims, period) if err != nil { - return nil, errors.New(fmt.Sprintf("Could not authorize: %s", err)) + return nil, fmt.Errorf("Could not authorize: %w", err) } return data, nil } -func AuthorizeCommand(client cloud.Client, account *account, claims *Claims) (Token, error) { +func AuthorizeCommand(client cloud.Client, account *account, claims *Claims, period *TokenPeriod) (Token, error) { when := time.Now().Unix() - found, ok := account.Cached(claims.Name, claims.Url) + found, ok := account.Cached(period, claims.Name, claims.Url) if ok { cached := make(Token) cached["endpoint"] = client.Endpoint() @@ -192,6 +185,7 @@ func AuthorizeCommand(client cloud.Client, account *account, claims *Claims) (To cached["token"] = found return cached, nil } + common.Timeline("authorize claim: %s (request)", claims.Name) body, err := claims.AsJson() if err != nil { return nil, err @@ -202,13 +196,13 @@ func AuthorizeCommand(client cloud.Client, account *account, claims *Claims) (To signed := HmacSignature(claims, account.Secret, nonce, bodyHash) request := client.NewRequest(claims.Url) request.Headers[contentType] = applicationJson - request.Headers[authorization] = RobocorpCloudHmac(account.Identifier, signed) + request.Headers[authorization] = ProductCloudHmac(account.Identifier, signed) request.Headers[nonceHeader] = nonce request.Headers[contentLength] = fmt.Sprintf("%d", size) request.Body = strings.NewReader(body) response := client.Post(request) if response.Status != 200 { - return nil, errors.New(fmt.Sprintf("%d: %s", response.Status, response.Body)) + return nil, fmt.Errorf("%d: %s", response.Status, response.Body) } token := make(Token) err = json.Unmarshal(response.Body, &token) @@ -222,8 +216,8 @@ func AuthorizeCommand(client cloud.Client, account *account, claims *Claims) (To account.WasVerified(when) trueToken, ok := token["token"].(string) if ok { - deadline := when + int64(3*(claims.ExpiresIn/4)) - account.CacheToken(claims.Name, claims.Url, trueToken, deadline) + account.CacheToken(claims.Name, claims.Url, trueToken, period.Deadline()) + common.Timeline("cached authorize claim: %s (new deadline: %d)", claims.Name, period.Deadline()) } return token, nil } @@ -231,33 +225,32 @@ func AuthorizeCommand(client cloud.Client, account *account, claims *Claims) (To func DeleteAccount(client cloud.Client, account *account) error { claims := DeleteClaims() bodyHash := Digest("{}") - when := time.Now().Unix() - nonce := fmt.Sprintf("%d", when) + nonce := fmt.Sprintf("%d", common.When) signed := HmacSignature(claims, account.Secret, nonce, bodyHash) request := client.NewRequest(claims.Url) request.Headers[contentType] = applicationJson - request.Headers[authorization] = RobocorpCloudHmac(account.Identifier, signed) + request.Headers[authorization] = ProductCloudHmac(account.Identifier, signed) request.Headers[nonceHeader] = nonce response := client.Delete(request) if response.Status < 200 || 299 < response.Status { - return errors.New(fmt.Sprintf("%d: %s", response.Status, response.Body)) + return fmt.Errorf("%d: %s", response.Status, response.Body) } return nil } func UserinfoCommand(client cloud.Client, account *account) (*UserInfo, error) { + when := time.Now().Unix() claims := VerificationClaims() bodyHash := Digest("{}") - when := time.Now().Unix() nonce := fmt.Sprintf("%d", when) signed := HmacSignature(claims, account.Secret, nonce, bodyHash) request := client.NewRequest(claims.Url) request.Headers[contentType] = applicationJson - request.Headers[authorization] = RobocorpCloudHmac(account.Identifier, signed) + request.Headers[authorization] = ProductCloudHmac(account.Identifier, signed) request.Headers[nonceHeader] = nonce response := client.Get(request) if response.Status != 200 { - return nil, errors.New(fmt.Sprintf("%d: %s", response.Status, response.Body)) + return nil, fmt.Errorf("%d: %s", response.Status, response.Body) } var result UserInfo err := json.Unmarshal(response.Body, &result) diff --git a/operations/authorize_test.go b/operations/authorize_test.go index 7cd30dbb..e2bd3662 100644 --- a/operations/authorize_test.go +++ b/operations/authorize_test.go @@ -2,7 +2,7 @@ package operations_test import ( "fmt" - "io/ioutil" + "io" "strings" "testing" @@ -45,7 +45,7 @@ func TestBodyIsCorrectlyConverted(t *testing.T) { reader := strings.NewReader("{\n}") wont_be.Nil(reader) - body, err := ioutil.ReadAll(reader) + body, err := io.ReadAll(reader) must_be.Nil(err) wont_be.Nil(body) must_be.Equal("{\n}", string(body)) @@ -58,11 +58,11 @@ func TestCanCreateBearerToken(t *testing.T) { must_be.Equal(operations.BearerToken("barbie"), "Bearer barbie") } -func TestCanCreateRobocorpCloudHmac(t *testing.T) { +func TestCanCreateProductCloudHmac(t *testing.T) { must_be, _ := hamlet.Specifications(t) - must_be.Equal(operations.RobocorpCloudHmac("11", "token"), "robocloud-hmac 11 token") - must_be.Equal(operations.RobocorpCloudHmac("1234", "abcd"), "robocloud-hmac 1234 abcd") + must_be.Equal(operations.ProductCloudHmac("11", "token"), "robocloud-hmac 11 token") + must_be.Equal(operations.ProductCloudHmac("1234", "abcd"), "robocloud-hmac 1234 abcd") } func TestCanCreateNewClaims(t *testing.T) { @@ -70,35 +70,23 @@ func TestCanCreateNewClaims(t *testing.T) { sut := operations.NewClaims("Mega", "https://some.com", 232) wont_be.Nil(sut) - must_be.Equal(len(sut.Capabilities), 0) - sut.Capabilities.Add("secret", true, true, false) - sut.Capabilities.Add("artifact", false, true, true) - sut.Capabilities.Add("livedata", false, true, true) - sut.Capabilities.Add("workitemdata", false, true, true) - sut.Capabilities.Add("workspace", true, false, false) - sut.Capabilities.Add("workspaceTree", true, true, false) - sut.Capabilities.Add("package", true, true, true) - must_be.Equal(len(sut.Capabilities), 7) + sut.CapabilitySet = "run/assistant" output, err := sut.AsJson() must_be.Nil(err) wont_be.Nil(output) - must_be.True(strings.Contains(output, "workspaceTree")) - must_be.True(strings.Contains(output, "true")) - must_be.True(strings.Contains(output, "false")) - must_be.True(strings.Contains(output, "list")) - must_be.True(strings.Contains(output, "read")) - must_be.True(strings.Contains(output, "write")) + must_be.True(strings.Contains(output, "capabilitySet")) + must_be.True(strings.Contains(output, "run/assistant")) } func TestCanCreateRobotClaims(t *testing.T) { must_be, wont_be := hamlet.Specifications(t) setup := operations.NewClaims("Robot", "https://some.com", 60) - setup.Capabilities.Add("package", true, true, true) + setup.CapabilitySet = "edit/robot" expected, err := setup.AsJson() must_be.Nil(err) - sut := operations.RobotClaims(60, "99") + sut := operations.EditRobotClaims(60, "99") wont_be.Nil(sut) result, err := sut.AsJson() must_be.Nil(err) @@ -106,18 +94,15 @@ func TestCanCreateRobotClaims(t *testing.T) { must_be.True(strings.Contains(sut.Url, "/workspaces/99/")) } -func TestCanCreateRunClaims(t *testing.T) { +func TestCanCreateRunRobotClaims(t *testing.T) { must_be, wont_be := hamlet.Specifications(t) setup := operations.NewClaims("Run", "https://some.com", 88) - setup.Capabilities.Add("secret", true, true, false) - setup.Capabilities.Add("artifact", false, false, true) - setup.Capabilities.Add("livedata", false, true, true) - setup.Capabilities.Add("workitemdata", false, true, true) + setup.CapabilitySet = "run/robot" expected, err := setup.AsJson() must_be.Nil(err) - sut := operations.RunClaims(88, "777") + sut := operations.RunRobotClaims(88, "777") wont_be.Nil(sut) result, err := sut.AsJson() must_be.Nil(err) @@ -146,16 +131,15 @@ func TestCanGetVerificationClaims(t *testing.T) { must_be.Equal("GET", sut.Method) } -func TestCanCreateWorkspaceTreeClaims(t *testing.T) { +func TestCanCreateViewWorkspacesClaims(t *testing.T) { must_be, wont_be := hamlet.Specifications(t) setup := operations.NewClaims("User", "https://some.com", 49) - setup.Capabilities.Add("workspace", true, false, false) - setup.Capabilities.Add("workspaceTree", true, true, false) + setup.CapabilitySet = "view/workspaces" expected, err := setup.AsJson() must_be.Nil(err) - sut := operations.WorkspaceTreeClaims(49) + sut := operations.ViewWorkspacesClaims(49) wont_be.Nil(sut) result, err := sut.AsJson() must_be.Nil(err) @@ -171,11 +155,11 @@ func TestCanCallAuthorizeCommand(t *testing.T) { wont_be.Nil(account) first := cloud.Response{Status: 200, Body: []byte("{\"token\":\"foo\",\"expiresIn\":1}")} client := mocks.NewClient(&first) - claims := operations.RunClaims(1, "777") - token, err := operations.AuthorizeCommand(client, account, claims) + claims := operations.RunRobotClaims(1, "777") + token, err := operations.AuthorizeCommand(client, account, claims, nil) must_be.Nil(err) wont_be.Nil(token) must_be.Equal(token["token"], "foo") - must_be.Equal(token["expiresIn"], 1.0) + //must_be.Equal(token["expiresIn"], 1.0) must_be.Equal(token["endpoint"], "https://this.is/mock") } diff --git a/operations/bugs_test.go b/operations/bugs_test.go new file mode 100644 index 00000000..d1789649 --- /dev/null +++ b/operations/bugs_test.go @@ -0,0 +1,26 @@ +package operations_test + +import ( + "testing" + + "github.com/robocorp/rcc/hamlet" + "github.com/robocorp/rcc/operations" +) + +func TestHashMatchingIsNotCaseSensitive(t *testing.T) { + must, wont := hamlet.Specifications(t) + + sut := operations.MetaTemplates{ + Hash: "\t\tCatsAndDogs\r\n", + } + + must.True(sut.MatchingHash(" catsanddogs ")) + wont.True(sut.MatchingHash(" dogsandcats ")) + + sut = operations.MetaTemplates{ + Hash: "catsanddogs", + } + + must.True(sut.MatchingHash(" CatsAndDogs ")) + wont.True(sut.MatchingHash(" dogsandcats ")) +} diff --git a/operations/cache.go b/operations/cache.go index 91f6ee0e..63eddfd8 100644 --- a/operations/cache.go +++ b/operations/cache.go @@ -4,21 +4,16 @@ import ( "fmt" "os" "path/filepath" + "strings" - "github.com/robocorp/rcc/conda" + "github.com/robocorp/rcc/common" "github.com/robocorp/rcc/pathlib" + "github.com/robocorp/rcc/set" "github.com/robocorp/rcc/xviper" "gopkg.in/yaml.v2" ) -type Folder struct { - Path string `yaml:"path" json:"robot"` - Created int64 `yaml:"created" json:"created"` - Updated int64 `yaml:"updated" json:"updated"` - Deleted int64 `yaml:"deleted" json:"deleted"` -} - type Credential struct { Account string `yaml:"account"` Context string `yaml:"context"` @@ -26,24 +21,38 @@ type Credential struct { Deadline int64 `yaml:"deadline"` } -type FolderMap map[string]*Folder type CredentialMap map[string]*Credential +type StampMap map[string]int64 type Cache struct { - Robots FolderMap `yaml:"robots"` Credentials CredentialMap `yaml:"credentials"` + Stamps StampMap `yaml:"stamps"` + Users []string `yaml:"users"` } func (it Cache) Ready() *Cache { - if it.Robots == nil { - it.Robots = make(FolderMap) - } if it.Credentials == nil { it.Credentials = make(CredentialMap) } + if it.Stamps == nil { + it.Stamps = make(StampMap) + } + if it.Users == nil { + it.Users = []string{} + } else { + it.Users = it.Userset() + } return &it } +func (it Cache) Userset() []string { + result := make([]string, 0, len(it.Users)) + for _, username := range it.Users { + result, _ = set.Update(result, strings.ToLower(username)) + } + return result +} + func cacheLockFile() string { return fmt.Sprintf("%s.lck", cacheLocation()) } @@ -53,23 +62,28 @@ func cacheLocation() string { if len(reference) > 0 { return filepath.Join(filepath.Dir(reference), "rcccache.yaml") } else { - return filepath.Join(conda.RobocorpHome(), "rcccache.yaml") + return filepath.Join(common.Product.Home(), "rcccache.yaml") } } func SummonCache() (*Cache, error) { var result Cache - locker, err := pathlib.Locker(cacheLockFile(), 125) + lockfile := cacheLockFile() + completed := pathlib.LockWaitMessage(lockfile, "Serialized cache access [cache lock]") + locker, err := pathlib.Locker(lockfile, 125, false) + completed() if err != nil { return nil, err } defer locker.Release() - source, err := os.Open(cacheLocation()) + cacheFile := cacheLocation() + source, err := os.Open(cacheFile) if err != nil { return result.Ready(), nil } defer source.Close() + defer pathlib.RestrictOwnerOnly(cacheFile) decoder := yaml.NewDecoder(source) err = decoder.Decode(&result) if err != nil { @@ -79,16 +93,25 @@ func SummonCache() (*Cache, error) { } func (it *Cache) Save() error { - locker, err := pathlib.Locker(cacheLockFile(), 125) + if common.WarrantyVoided() { + return nil + } + lockfile := cacheLockFile() + completed := pathlib.LockWaitMessage(lockfile, "Serialized cache access [cache lock]") + locker, err := pathlib.Locker(lockfile, 125, false) + completed() if err != nil { return err } defer locker.Release() - sink, err := os.Create(cacheLocation()) + cacheFile := cacheLocation() + sink, err := pathlib.Create(cacheFile) if err != nil { return err } + defer sink.Close() + defer pathlib.RestrictOwnerOnly(cacheFile) encoder := yaml.NewEncoder(sink) err = encoder.Encode(it) if err != nil { diff --git a/operations/carrier.go b/operations/carrier.go new file mode 100644 index 00000000..6319fdb5 --- /dev/null +++ b/operations/carrier.go @@ -0,0 +1,173 @@ +package operations + +import ( + "encoding/binary" + "fmt" + "io" + "os" + "path/filepath" + + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/pathlib" +) + +const ( + scissors = `---8x---` +) + +func FindExecutable() (string, error) { + self, err := os.Executable() + if err != nil { + return "", err + } + self, err = filepath.EvalSymlinks(self) + if err != nil { + return "", err + } + self, err = filepath.Abs(self) + if err != nil { + return "", err + } + return self, nil +} + +func SelfCopy(target string) error { + self, err := FindExecutable() + if err != nil { + return err + } + source, err := os.Open(self) + if err != nil { + return err + } + defer source.Close() + + sink, err := os.OpenFile(target, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0o755) + if err != nil { + return err + } + defer sink.Close() + size, err := io.Copy(sink, source) + if err != nil { + return err + } + common.Debug("Copied %q to %q as size of %d bytes.", self, target, size) + return nil +} + +func SelfAppend(target, payload string) error { + size, ok := pathlib.Size(target) + if !ok { + return fmt.Errorf("Could not get size of %q.", target) + } + source, err := os.Open(payload) + if err != nil { + return err + } + defer source.Close() + sink, err := os.OpenFile(target, os.O_WRONLY|os.O_APPEND, 0o755) + if err != nil { + return err + } + defer sink.Close() + _, err = sink.Write([]byte(scissors)) + if err != nil { + return err + } + _, err = io.Copy(sink, source) + if err != nil { + return err + } + err = binary.Write(sink, binary.LittleEndian, size) + if err != nil { + return err + } + return nil +} + +func HasPayload(filename string) (bool, error) { + reader, err := PayloadReaderAt(filename) + if err != nil { + return false, err + } + reader.Close() + return true, nil +} + +func IsCarrier() (bool, error) { + carrier, err := FindExecutable() + if err != nil { + return false, err + } + return HasPayload(carrier) +} + +type ReaderCloserAt interface { + io.ReaderAt + io.Closer + Limit() int64 +} + +type carrier struct { + source *os.File + offset int64 + limit int64 +} + +func (it *carrier) Limit() int64 { + return it.limit +} + +func (it *carrier) ReadAt(target []byte, offset int64) (int, error) { + _, err := it.source.Seek(it.offset+offset, 0) + if err != nil { + return 0, err + } + return it.source.Read(target) +} + +func (it *carrier) Close() error { + return it.source.Close() +} + +func PayloadReaderAt(filename string) (ReaderCloserAt, error) { + size, ok := pathlib.Size(filename) + if !ok { + return nil, fmt.Errorf("Could not get size of %q.", filename) + } + source, err := os.Open(filename) + if err != nil { + return nil, err + } + _, err = source.Seek(size-8, 0) + if err != nil { + source.Close() + return nil, err + } + var offset int64 + err = binary.Read(source, binary.LittleEndian, &offset) + if err != nil { + source.Close() + return nil, err + } + if offset < 0 || size <= offset { + source.Close() + return nil, fmt.Errorf("%q has no carrier payload.", filename) + } + _, err = source.Seek(offset, 0) + if err != nil { + source.Close() + return nil, err + } + marker := make([]byte, 8) + count, err := source.Read(marker) + if err != nil { + source.Close() + return nil, err + } + if count != 8 || string(marker) != scissors { + source.Close() + return nil, fmt.Errorf("%q has no carrier payload.", filename) + } + return &carrier{source, offset + 8, size - offset - 16}, nil +} diff --git a/operations/carrier_test.go b/operations/carrier_test.go new file mode 100644 index 00000000..63676d6e --- /dev/null +++ b/operations/carrier_test.go @@ -0,0 +1,40 @@ +package operations_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/robocorp/rcc/hamlet" + "github.com/robocorp/rcc/operations" + "github.com/robocorp/rcc/pathlib" +) + +func TestCanUseCarrier(t *testing.T) { + must, wont := hamlet.Specifications(t) + + tempFile := filepath.Join(os.TempDir(), "carrier") + if pathlib.Exists(tempFile) { + os.Remove(tempFile) + } + + wont.True(pathlib.Exists(tempFile)) + must.Nil(operations.SelfCopy(tempFile)) + must.True(pathlib.Exists(tempFile)) + must.Nil(operations.SelfCopy(tempFile)) + must.True(pathlib.Exists(tempFile)) + + original, ok := pathlib.Size(tempFile) + must.True(ok) + + must.Nil(operations.SelfAppend(tempFile, "testdata/payload.txt")) + + final, ok := pathlib.Size(tempFile) + must.True(ok) + + must.Equal(original+24, final) + + ok, err := operations.HasPayload(tempFile) + must.Nil(err) + must.True(ok) +} diff --git a/operations/community.go b/operations/community.go index 368e49f0..2bed8a2e 100644 --- a/operations/community.go +++ b/operations/community.go @@ -2,15 +2,15 @@ package operations import ( "crypto/sha256" - "errors" "fmt" "io" "net/http" - "os" "regexp" "strings" "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/pathlib" + "github.com/robocorp/rcc/settings" ) const ( @@ -47,17 +47,18 @@ func CommunityLocation(name, branch string) string { } func DownloadCommunityRobot(url, filename string) error { - response, err := http.Get(url) + client := &http.Client{Transport: settings.Global.ConfiguredHttpTransport()} + response, err := client.Get(url) if err != nil { return err } defer response.Body.Close() if response.StatusCode < 200 || 299 < response.StatusCode { - return errors.New(fmt.Sprintf("%s (%s)", response.Status, url)) + return fmt.Errorf("%s (%s)", response.Status, url) } - out, err := os.Create(filename) + out, err := pathlib.Create(filename) if err != nil { return err } @@ -73,7 +74,7 @@ func DownloadCommunityRobot(url, filename string) error { return err } - if common.DebugFlag { + if common.DebugFlag() { sum := fmt.Sprintf("%02x", digest.Sum(nil)) common.Debug("SHA256 sum: %s", sum) } diff --git a/operations/credentials.go b/operations/credentials.go index 58b628a2..66e138c2 100644 --- a/operations/credentials.go +++ b/operations/credentials.go @@ -1,7 +1,6 @@ package operations import ( - "encoding/json" "fmt" "regexp" "sort" @@ -10,6 +9,8 @@ import ( "github.com/robocorp/rcc/cloud" "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/pretty" + "github.com/robocorp/rcc/settings" "github.com/robocorp/rcc/xviper" ) @@ -83,6 +84,10 @@ func VerifyAccounts(force bool) { } } +func (it *account) IsValid() bool { + return len(it.Endpoint) > 0 && len(it.Account) > 0 +} + func (it *account) CacheKey() string { return fmt.Sprintf("%s.%s", it.Identifier, it.Secret[:6]) } @@ -106,7 +111,7 @@ func (it *account) CacheToken(name, url, token string, deadline int64) { cache.Credentials[fullkey] = &credential } -func (it *account) Cached(name, url string) (string, bool) { +func (it *account) Cached(period *TokenPeriod, name, url string) (string, bool) { if common.NoCache { return "", false } @@ -119,13 +124,15 @@ func (it *account) Cached(name, url string) (string, bool) { if !ok { return "", false } - if found.Deadline < time.Now().Unix() { + liveline := period.Liveline() + if found.Deadline < liveline { return "", false } + common.Timeline("using cached token: %s (%d < %d)", name, liveline, found.Deadline) return found.Token, true } -func (it *account) Delete() error { +func (it *account) Delete(timeout time.Duration) error { prefix := accountsPrefix + it.Account defer xviper.Set(prefix, "deleted") @@ -133,6 +140,7 @@ func (it *account) Delete() error { if err != nil { return err } + client = client.WithTimeout(timeout) return DeleteAccount(client, it) } @@ -186,11 +194,6 @@ func ListAccounts(json bool) { } } -func EncodeCredentials(target *json.Encoder, force bool) error { - VerifyAccounts(force) - return target.Encode(smudgeSecrets(findAccounts())) -} - func loadAccount(label string) *account { prefix := accountsPrefix + label var details Token @@ -213,9 +216,9 @@ func loadAccount(label string) *account { } func createEphemeralAccount(parts []string) *account { - BackgroundMetric("rcc", "rcc.account.ephemeral", common.Version) + cloud.InternalBackgroundMetric(common.ControllerIdentity(), "rcc.account.ephemeral", common.Version) common.NoCache = true - endpoint := common.DefaultEndpoint + endpoint := settings.Global.DefaultEndpoint() if len(parts[3]) > 0 { endpoint = parts[3] } @@ -235,11 +238,12 @@ func AccountByName(label string) *account { if dynamic != nil { return createEphemeralAccount(dynamic) } + pretty.Guard(xviper.IsAvailable(), 1, "This rcc is not configured yet. Please, fix that first.") if len(label) == 0 { label = DefaultAccountName() } found := loadAccount(label) - if found.Account == label { + if found.Account == label && found.IsValid() { return found } return nil diff --git a/operations/credentials_test.go b/operations/credentials_test.go index 464e57d1..7a32712c 100644 --- a/operations/credentials_test.go +++ b/operations/credentials_test.go @@ -5,10 +5,11 @@ import ( "path/filepath" "strings" "testing" + "time" - "github.com/robocorp/rcc/common" "github.com/robocorp/rcc/hamlet" "github.com/robocorp/rcc/operations" + "github.com/robocorp/rcc/settings" "github.com/robocorp/rcc/xviper" ) @@ -20,7 +21,7 @@ func TestCanGetEphemeralDefaultEndpointAccountByName(t *testing.T) { wont_be.Nil(sut) must_be.Equal("Ephemeral", sut.Account) must_be.Equal("1111", sut.Identifier) - must_be.Equal(common.DefaultEndpoint, sut.Endpoint) + must_be.Equal(settings.Global.DefaultEndpoint(), sut.Endpoint) must_be.Equal("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", sut.Secret) wont_be.True(sut.Default) } @@ -66,7 +67,7 @@ func TestCanCreateAndDeleteAccount(t *testing.T) { wont_be.Nil(sut) must_be.True(strings.HasSuffix(xviper.ConfigFileUsed(), "rcctest.yaml")) must_be.Equal("42.long_a", sut.CacheKey()) - sut.Delete() + sut.Delete(50 * time.Millisecond) sut = operations.AccountByName("dele") must_be.Nil(sut) } diff --git a/operations/diagnostics.go b/operations/diagnostics.go new file mode 100644 index 00000000..92a459b3 --- /dev/null +++ b/operations/diagnostics.go @@ -0,0 +1,710 @@ +package operations + +import ( + "encoding/json" + "fmt" + "io" + "net" + "os" + "os/user" + "path/filepath" + "runtime" + "sort" + "strings" + "time" + + "github.com/robocorp/rcc/cloud" + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/conda" + "github.com/robocorp/rcc/htfs" + "github.com/robocorp/rcc/journal" + "github.com/robocorp/rcc/pathlib" + "github.com/robocorp/rcc/pretty" + "github.com/robocorp/rcc/robot" + "github.com/robocorp/rcc/settings" + "github.com/robocorp/rcc/xviper" + "gopkg.in/yaml.v2" +) + +const ( + canaryUrl = `/canary.txt` + pypiCanaryUrl = `/jupyterlab-pygments/` + condaCanaryUrl = `/conda-forge/linux-64/repodata.json` + statusOk = `ok` + statusWarning = `warning` + statusFail = `fail` + statusFatal = `fatal` +) + +var ( + ignorePathContains = []string{".vscode", ".ipynb_checkpoints", ".virtual_documents"} +) + +func shouldIgnorePath(fullpath string) bool { + lowpath := strings.ToLower(fullpath) + for _, ignore := range ignorePathContains { + if strings.Contains(lowpath, ignore) { + return true + } + } + return false +} + +type stringerr func() (string, error) + +func justText(source stringerr) string { + result, _ := source() + return result +} + +func runDiagnostics(quick bool) *common.DiagnosticStatus { + result := &common.DiagnosticStatus{ + Details: make(map[string]string), + Checks: []*common.DiagnosticCheck{}, + } + result.Details["executable"] = common.BinRcc() + result.Details["rcc"] = common.Version + result.Details["rcc.bin"] = common.BinRcc() + result.Details["micromamba"] = conda.MicromambaVersion() + result.Details["micromamba.bin"] = conda.BinMicromamba() + result.Details[common.Product.HomeVariable()] = common.Product.Home() + result.Details["ROBOCORP_OVERRIDE_SYSTEM_REQUIREMENTS"] = fmt.Sprintf("%v", common.OverrideSystemRequirements()) + result.Details["RCC_VERBOSE_ENVIRONMENT_BUILDING"] = fmt.Sprintf("%v", common.VerboseEnvironmentBuilding()) + result.Details["RCC_REMOTE_ORIGIN"] = fmt.Sprintf("%v", common.RccRemoteOrigin()) + who, _ := user.Current() + result.Details["user-name"] = who.Name + result.Details["user-username"] = who.Username + result.Details["user-cache-dir"] = justText(os.UserCacheDir) + result.Details["user-config-dir"] = justText(os.UserConfigDir) + result.Details["user-home-dir"] = justText(os.UserHomeDir) + result.Details["working-dir"] = justText(os.Getwd) + result.Details["hostname"] = justText(os.Hostname) + result.Details["tempdir"] = os.TempDir() + result.Details["controller"] = common.ControllerIdentity() + result.Details["user-agent"] = common.UserAgent() + result.Details["installationId"] = xviper.TrackingIdentity() + result.Details["telemetry-enabled"] = fmt.Sprintf("%v", xviper.CanTrack()) + result.Details["config-piprc-used"] = fmt.Sprintf("%v", settings.Global.HasPipRc()) + result.Details["config-micromambarc-used"] = fmt.Sprintf("%v", settings.Global.HasMicroMambaRc()) + result.Details["config-settings-yaml-used"] = fmt.Sprintf("%v", pathlib.IsFile(common.SettingsFile())) + result.Details["config-settings-yaml-age-seconds"] = fmt.Sprintf("%d", pathlib.Age(common.SettingsFile())) + result.Details["config-active-profile"] = settings.Global.Name() + result.Details["config-https-proxy"] = settings.Global.HttpsProxy() + result.Details["config-http-proxy"] = settings.Global.HttpProxy() + result.Details["config-no-proxy"] = settings.Global.NoProxy() + result.Details["config-ssl-verify"] = fmt.Sprintf("%v", settings.Global.VerifySsl()) + result.Details["config-ssl-no-revoke"] = fmt.Sprintf("%v", settings.Global.NoRevocation()) + result.Details["config-legacy-renegotiation-allowed"] = fmt.Sprintf("%v", settings.Global.LegacyRenegotiation()) + result.Details["os-holo-location"] = common.Product.HoloLocation() + result.Details["hololib-location"] = common.HololibLocation() + result.Details["hololib-catalog-location"] = common.HololibCatalogLocation() + result.Details["hololib-library-location"] = common.HololibLibraryLocation() + result.Details["holotree-location"] = common.HolotreeLocation() + result.Details["holotree-shared"] = fmt.Sprintf("%v", common.SharedHolotree) + result.Details["holotree-global-shared"] = fmt.Sprintf("%v", pathlib.IsFile(common.SharedMarkerLocation())) + result.Details["holotree-user-id"] = common.UserHomeIdentity() + result.Details["os"] = common.Platform() + result.Details["os-details"] = settings.OperatingSystem() + result.Details["cpus"] = fmt.Sprintf("%d", runtime.NumCPU()) + result.Details["when"] = time.Now().Format(time.RFC3339 + " (MST)") + result.Details["timezone"] = time.Now().Format("MST") + result.Details["no-build"] = fmt.Sprintf("%v", settings.Global.NoBuild()) + result.Details["ENV:ComSpec"] = os.Getenv("ComSpec") + result.Details["ENV:SHELL"] = os.Getenv("SHELL") + result.Details["ENV:LANG"] = os.Getenv("LANG") + result.Details["warranty-voided-mode"] = fmt.Sprintf("%v", common.WarrantyVoided()) + result.Details["temp-management-disabled"] = fmt.Sprintf("%v", common.DisableTempManagement()) + result.Details["pyc-management-disabled"] = fmt.Sprintf("%v", common.DisablePycManagement()) + result.Details["is-bundled"] = fmt.Sprintf("%v", common.IsBundled()) + + for name, filename := range lockfiles() { + result.Details[name] = filename + } + + who, err := user.Current() + if err == nil { + result.Details["uid:gid"] = fmt.Sprintf("%s:%s", who.Uid, who.Gid) + } + + // checks + if common.SharedHolotree { + result.Checks = append(result.Checks, verifySharedDirectory(common.Product.HoloLocation())) + result.Checks = append(result.Checks, verifySharedDirectory(common.HololibLocation())) + result.Checks = append(result.Checks, verifySharedDirectory(common.HololibCatalogLocation())) + result.Checks = append(result.Checks, verifySharedDirectory(common.HololibLibraryLocation())) + } + result.Checks = append(result.Checks, productHomeCheck()) + check := productHomeMemberCheck() + if check != nil { + result.Checks = append(result.Checks, check) + } + check = workdirCheck() + if check != nil { + result.Checks = append(result.Checks, check) + } + + result.Checks = append(result.Checks, anyPathCheck("CURL_CA_BUNDLE")) + result.Checks = append(result.Checks, anyPathCheck("NODE_EXTRA_CA_CERTS")) + result.Checks = append(result.Checks, anyPathCheck("NODE_OPTIONS")) + result.Checks = append(result.Checks, anyPathCheck("NODE_PATH")) + result.Checks = append(result.Checks, anyPathCheck("NODE_TLS_REJECT_UNAUTHORIZED")) + result.Checks = append(result.Checks, anyPathCheck("PIP_CONFIG_FILE")) + result.Checks = append(result.Checks, anyPathCheck("PLAYWRIGHT_BROWSERS_PATH")) + result.Checks = append(result.Checks, anyPathCheck("PYTHONPATH")) + result.Checks = append(result.Checks, anyPathCheck("REQUESTS_CA_BUNDLE")) + result.Checks = append(result.Checks, anyPathCheck("SSL_CERT_DIR")) + result.Checks = append(result.Checks, anyPathCheck("SSL_CERT_FILE")) + result.Checks = append(result.Checks, anyPathCheck("WDM_SSL_VERIFY")) + result.Checks = append(result.Checks, anyPathCheck("VIRTUAL_ENV")) + + result.Checks = append(result.Checks, anyEnvVarCheck("RCC_NO_TEMP_MANAGEMENT")) + result.Checks = append(result.Checks, anyEnvVarCheck("RCC_NO_PYC_MANAGEMENT")) + result.Checks = append(result.Checks, anyEnvVarCheck("ROBOCORP_OVERRIDE_SYSTEM_REQUIREMENTS")) + + if !common.OverrideSystemRequirements() { + result.Checks = append(result.Checks, longPathSupportCheck()) + } + result.Checks = append(result.Checks, lockpidsCheck()...) + result.Checks = append(result.Checks, lockfilesCheck()...) + if quick { + return result + } + + // Move slow checks below this position + + hostnames := settings.Global.Hostnames() + dnsStopwatch := common.Stopwatch("DNS lookup time for %d hostnames was about", len(hostnames)) + for _, host := range hostnames { + result.Checks = append(result.Checks, dnsLookupCheck(host)) + } + result.Details["dns-lookup-time"] = dnsStopwatch.Text() + tlsStopwatch := common.Stopwatch("TLS verification time for %d hostnames was about", len(hostnames)) + tlsRoots := make(map[string]bool) + for _, host := range hostnames { + result.Checks = append(result.Checks, tlsCheckHost(host, tlsRoots)...) + } + result.Details["tls-lookup-time"] = tlsStopwatch.Text() + if len(hostnames) > 1 && len(tlsRoots) == 1 { + for name, _ := range tlsRoots { + result.Details["tls-proxy-firewall"] = name + } + } else { + result.Details["tls-proxy-firewall"] = "undetectable" + } + result.Checks = append(result.Checks, canaryDownloadCheck()) + result.Checks = append(result.Checks, pypiHeadCheck()) + result.Checks = append(result.Checks, condaHeadCheck()) + return result +} + +func lockfiles() map[string]string { + result := make(map[string]string) + result["lock-config"] = xviper.Lockfile() + result["lock-cache"] = cacheLockFile() + result["lock-holotree"] = common.HolotreeLock() + result["lock-robocorp"] = common.ProductLock() + result["lock-userlock"] = htfs.UserHolotreeLockfile() + return result +} + +func longPathSupportCheck() *common.DiagnosticCheck { + supportLongPathUrl := settings.Global.DocsLink("troubleshooting/windows-long-path") + if conda.HasLongPathSupport() { + return &common.DiagnosticCheck{ + Type: "OS", + Category: common.CategoryLongPath, + Status: statusOk, + Message: "Supports long enough paths.", + Link: supportLongPathUrl, + } + } + return &common.DiagnosticCheck{ + Type: "OS", + Category: common.CategoryLongPath, + Status: statusFail, + Message: "Does not support long path names!", + Link: supportLongPathUrl, + } +} + +func lockfilesCheck() []*common.DiagnosticCheck { + content := []byte(fmt.Sprintf("lock check %s @%d", common.Version, common.When)) + files := lockfiles() + count := len(files) + result := make([]*common.DiagnosticCheck, 0, count) + support := settings.Global.DocsLink("troubleshooting") + failed := false + for identity, filename := range files { + if !pathlib.Exists(filepath.Dir(filename)) { + common.Trace("Wont check lock writing on %q (%s), since directory does not exist.", filename, identity) + continue + } + err := os.WriteFile(filename, content, 0o666) + if err != nil { + result = append(result, &common.DiagnosticCheck{ + Type: "OS", + Category: common.CategoryLockFile, + Status: statusFail, + Message: fmt.Sprintf("Lock file %q write failed, reason: %v", identity, err), + Link: support, + }) + failed = true + } + } + if !failed { + result = append(result, &common.DiagnosticCheck{ + Type: "OS", + Category: common.CategoryLockFile, + Status: statusOk, + Message: fmt.Sprintf("%d lockfiles all seem to work correctly (for this user).", count), + Link: support, + }) + } + return result +} + +func lockpidsCheck() []*common.DiagnosticCheck { + support := settings.Global.DocsLink("troubleshooting") + result := []*common.DiagnosticCheck{} + entries, err := pathlib.LoadLockpids() + if err != nil { + result = append(result, &common.DiagnosticCheck{ + Type: "OS", + Category: common.CategoryLockPid, + Status: statusWarning, + Message: fmt.Sprintf("Problem loading lock pids, reason: %v", err), + Link: support, + }) + return result + } + pid := os.Getpid() + for _, entry := range entries { + level := statusWarning + if entry.ProcessID == pid { + level = statusOk + } + result = append(result, &common.DiagnosticCheck{ + Type: "OS", + Category: common.CategoryLockPid, + Status: level, + Message: entry.Message(), + Link: support, + }) + } + if len(result) == 0 { + result = append(result, &common.DiagnosticCheck{ + Type: "OS", + Category: common.CategoryLockPid, + Status: statusOk, + Message: "No pending lock files detected.", + Link: support, + }) + } + return result +} + +func anyEnvVarCheck(key string) *common.DiagnosticCheck { + supportGeneralUrl := settings.Global.DocsLink("troubleshooting") + anyVar := os.Getenv(key) + if len(anyVar) > 0 { + return &common.DiagnosticCheck{ + Type: "OS", + Category: common.CategoryEnvVarCheck, + Status: statusWarning, + Message: fmt.Sprintf("%s is set to %q. This may cause problems.", key, anyVar), + Link: supportGeneralUrl, + } + } + return &common.DiagnosticCheck{ + Type: "OS", + Category: common.CategoryEnvVarCheck, + Status: statusOk, + Message: fmt.Sprintf("%s is not set, which is good.", key), + Link: supportGeneralUrl, + } +} + +func anyPathCheck(key string) *common.DiagnosticCheck { + supportGeneralUrl := settings.Global.DocsLink("troubleshooting") + anyPath := os.Getenv(key) + if len(anyPath) > 0 { + return &common.DiagnosticCheck{ + Type: "OS", + Category: common.CategoryPathCheck, + Status: statusWarning, + Message: fmt.Sprintf("%s is set to %q. This may cause problems.", key, anyPath), + Link: supportGeneralUrl, + } + } + return &common.DiagnosticCheck{ + Type: "OS", + Category: common.CategoryPathCheck, + Status: statusOk, + Message: fmt.Sprintf("%s is not set, which is good.", key), + Link: supportGeneralUrl, + } +} + +func verifySharedDirectory(fullpath string) *common.DiagnosticCheck { + shared := pathlib.IsSharedDir(fullpath) + supportGeneralUrl := settings.Global.DocsLink("troubleshooting") + if !shared { + return &common.DiagnosticCheck{ + Type: "OS", + Category: common.CategoryHolotreeShared, + Status: statusWarning, + Message: fmt.Sprintf("%q is not shared. This may cause problems.", fullpath), + Link: supportGeneralUrl, + } + } + return &common.DiagnosticCheck{ + Type: "OS", + Category: common.CategoryHolotreeShared, + Status: statusOk, + Message: fmt.Sprintf("%q is shared, which is ok.", fullpath), + Link: supportGeneralUrl, + } +} + +func workdirCheck() *common.DiagnosticCheck { + supportGeneralUrl := settings.Global.DocsLink("troubleshooting") + workarea, err := os.Getwd() + if err != nil { + return nil + } + inside, err := common.IsInsideProductHome(workarea) + if err != nil { + return nil + } + if inside { + return &common.DiagnosticCheck{ + Type: "RPA", + Category: common.CategoryPathCheck, + Status: statusWarning, + Message: fmt.Sprintf("Working directory %q is inside %s (%s).", workarea, common.Product.HomeVariable(), common.Product.Home()), + Link: supportGeneralUrl, + } + } + return nil +} + +func productHomeMemberCheck() *common.DiagnosticCheck { + supportGeneralUrl := settings.Global.DocsLink("troubleshooting") + cache, err := SummonCache() + if err != nil || len(cache.Users) < 2 { + return nil + } + members := strings.Join(cache.Users, ", ") + return &common.DiagnosticCheck{ + Type: "RPA", + Category: common.CategoryProductHomeMembers, + Status: statusWarning, + Message: fmt.Sprintf("More than one user is sharing %s (%s). Those users are: %s.", common.Product.HomeVariable(), common.Product.Home(), members), + Link: supportGeneralUrl, + } +} + +func productHomeCheck() *common.DiagnosticCheck { + supportGeneralUrl := settings.Global.DocsLink("troubleshooting") + if !conda.ValidLocation(common.Product.Home()) { + return &common.DiagnosticCheck{ + Type: "RPA", + Category: common.CategoryProductHome, + Status: statusFatal, + Message: fmt.Sprintf("%s (%s) contains characters that makes RPA fail.", common.Product.HomeVariable(), common.Product.Home()), + Link: supportGeneralUrl, + } + } + userhome, err := os.UserHomeDir() + if err == nil { + inside, err := common.IsInsideProductHome(userhome) + if err == nil && inside { + return &common.DiagnosticCheck{ + Type: "RPA", + Category: common.CategoryProductHome, + Status: statusWarning, + Message: fmt.Sprintf("User home directory %q is inside %s (%s).", userhome, common.Product.HomeVariable(), common.Product.Home()), + Link: supportGeneralUrl, + } + } + } + return &common.DiagnosticCheck{ + Type: "RPA", + Category: common.CategoryProductHome, + Status: statusOk, + Message: fmt.Sprintf("%s (%s) is good enough.", common.Product.HomeVariable(), common.Product.Home()), + Link: supportGeneralUrl, + } +} + +func dnsLookupCheck(site string) *common.DiagnosticCheck { + supportNetworkUrl := settings.Global.DocsLink("troubleshooting/firewall-and-proxies") + found, err := net.LookupHost(site) + if err != nil { + return &common.DiagnosticCheck{ + Type: "network", + Category: common.CategoryNetworkDNS, + Status: statusFail, + Message: fmt.Sprintf("DNS lookup %q failed: %v", site, err), + Link: supportNetworkUrl, + } + } + return &common.DiagnosticCheck{ + Type: "network", + Category: common.CategoryNetworkDNS, + Status: statusOk, + Message: fmt.Sprintf("%s found [DNS query]: %v", site, found), + Link: supportNetworkUrl, + } +} + +func condaHeadCheck() *common.DiagnosticCheck { + supportNetworkUrl := settings.Global.DocsLink("troubleshooting/firewall-and-proxies") + client, err := cloud.NewClient(settings.Global.CondaLink("")) + if err != nil { + return &common.DiagnosticCheck{ + Type: "network", + Category: common.CategoryNetworkLink, + Status: statusWarning, + Message: fmt.Sprintf("%v: %v", settings.Global.CondaLink(""), err), + Link: supportNetworkUrl, + } + } + request := client.NewRequest(condaCanaryUrl) + response := client.Head(request) + if response.Status >= 400 { + return &common.DiagnosticCheck{ + Type: "network", + Category: common.CategoryNetworkHEAD, + Status: statusWarning, + Message: fmt.Sprintf("Conda canary download failed: %d %v", response.Status, response.Err), + Link: supportNetworkUrl, + } + } + return &common.DiagnosticCheck{ + Type: "network", + Category: common.CategoryNetworkHEAD, + Status: statusOk, + Message: fmt.Sprintf("Conda canary download successful [HEAD request]: %s", settings.Global.CondaLink(condaCanaryUrl)), + Link: supportNetworkUrl, + } +} + +func pypiHeadCheck() *common.DiagnosticCheck { + supportNetworkUrl := settings.Global.DocsLink("troubleshooting/firewall-and-proxies") + client, err := cloud.NewClient(settings.Global.PypiLink("")) + if err != nil { + return &common.DiagnosticCheck{ + Type: "network", + Category: common.CategoryNetworkLink, + Status: statusWarning, + Message: fmt.Sprintf("%v: %v", settings.Global.PypiLink(""), err), + Link: supportNetworkUrl, + } + } + request := client.NewRequest(pypiCanaryUrl) + response := client.Head(request) + if response.Status >= 400 { + return &common.DiagnosticCheck{ + Type: "network", + Category: common.CategoryNetworkHEAD, + Status: statusWarning, + Message: fmt.Sprintf("PyPI canary download failed: %d %v", response.Status, response.Err), + Link: supportNetworkUrl, + } + } + return &common.DiagnosticCheck{ + Type: "network", + Category: common.CategoryNetworkHEAD, + Status: statusOk, + Message: fmt.Sprintf("PyPI canary download successful [HEAD request]: %s", settings.Global.PypiLink(pypiCanaryUrl)), + Link: supportNetworkUrl, + } +} + +func canaryDownloadCheck() *common.DiagnosticCheck { + supportNetworkUrl := settings.Global.DocsLink("troubleshooting/firewall-and-proxies") + client, err := cloud.NewClient(settings.Global.DownloadsLink("")) + if err != nil { + return &common.DiagnosticCheck{ + Type: "network", + Category: common.CategoryNetworkLink, + Status: statusFail, + Message: fmt.Sprintf("%v: %v", settings.Global.DownloadsLink(""), err), + Link: supportNetworkUrl, + } + } + request := client.NewRequest(canaryUrl) + response := client.Get(request) + if response.Status != 200 || string(response.Body) != "Used to testing connections" { + return &common.DiagnosticCheck{ + Type: "network", + Category: common.CategoryNetworkCanary, + Status: statusFail, + Message: fmt.Sprintf("Canary download failed: %d: %v %s", response.Status, response.Err, response.Body), + Link: supportNetworkUrl, + } + } + return &common.DiagnosticCheck{ + Type: "network", + Category: common.CategoryNetworkCanary, + Status: statusOk, + Message: fmt.Sprintf("Canary download successful [GET request]: %s", settings.Global.DownloadsLink(canaryUrl)), + Link: supportNetworkUrl, + } +} + +func jsonDiagnostics(sink io.Writer, details *common.DiagnosticStatus) { + form, err := details.AsJson() + if err != nil { + pretty.Exit(1, "Error: %s", err) + } + fmt.Fprintln(sink, form) +} + +func humaneDiagnostics(sink io.Writer, details *common.DiagnosticStatus, showStatistics bool) { + fmt.Fprintln(sink, "Diagnostics:") + keys := make([]string, 0, len(details.Details)) + for key, _ := range details.Details { + keys = append(keys, key) + } + sort.Strings(keys) + for _, key := range keys { + value := details.Details[key] + fmt.Fprintf(sink, " - %-38s... %q\n", key, value) + } + fmt.Fprintln(sink, "") + fmt.Fprintln(sink, "Checks:") + for _, check := range details.Checks { + fmt.Fprintf(sink, " - %-8s %-8s %s\n", check.Type, check.Status, check.Message) + } + if !showStatistics { + return + } + count, body := journal.MakeStatistics(12, false, false, false, false) + if count > 4 { + fmt.Fprintln(sink, "") + fmt.Fprintln(sink, "Statistics:") + fmt.Fprintln(sink, "") + fmt.Fprintln(sink, string(body)) + } +} + +func fileIt(filename string) (io.WriteCloser, error) { + if len(filename) == 0 { + return os.Stdout, nil + } + file, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644) + if err != nil { + return nil, err + } + return file, nil +} + +func ProduceNetDiagnostics(body []byte, json bool) (*common.DiagnosticStatus, error) { + config, err := parseNetworkDiagnosticConfig(body) + if err != nil { + return nil, err + } + result := &common.DiagnosticStatus{ + Details: make(map[string]string), + Checks: []*common.DiagnosticCheck{}, + } + networkDiagnostics(config, result) + if json { + jsonDiagnostics(os.Stdout, result) + } else { + humaneDiagnostics(os.Stdout, result, false) + } + return nil, nil +} + +func ProduceDiagnostics(filename, robotfile string, json, production, quick bool) (*common.DiagnosticStatus, error) { + file, err := fileIt(filename) + if err != nil { + return nil, err + } + defer file.Close() + result := runDiagnostics(quick) + if len(robotfile) > 0 { + addRobotDiagnostics(robotfile, result, production) + } + settings.Global.Diagnostics(result) + if json { + jsonDiagnostics(file, result) + } else { + humaneDiagnostics(file, result, true) + } + return result, nil +} + +type Unmarshaler func([]byte, interface{}) error + +func diagnoseFilesUnmarshal(tool Unmarshaler, label, rootdir string, paths []string, target *common.DiagnosticStatus) { + supportGeneralUrl := settings.Global.DocsLink("troubleshooting") + target.Details[fmt.Sprintf("%s-file-count", strings.ToLower(label))] = fmt.Sprintf("%d file(s)", len(paths)) + diagnose := target.Diagnose(label) + var canary interface{} + success := true + investigated := false + for _, tail := range paths { + investigated = true + fullpath := filepath.Join(rootdir, tail) + if shouldIgnorePath(fullpath) { + continue + } + content, err := os.ReadFile(fullpath) + if err != nil { + diagnose.Fail(0, supportGeneralUrl, "Problem reading %s file %q: %v", label, tail, err) + success = false + continue + } + err = tool(content, &canary) + if err != nil { + diagnose.Fail(0, supportGeneralUrl, "Problem parsing %s file %q: %v", label, tail, err) + success = false + } + } + if investigated && success { + diagnose.Ok(0, "%s files are readable and can be parsed.", label) + } +} + +func addFileDiagnostics(rootdir string, target *common.DiagnosticStatus) { + jsons := pathlib.RecursiveGlob(rootdir, "*.json") + diagnoseFilesUnmarshal(json.Unmarshal, "JSON", rootdir, jsons, target) + yamls := pathlib.RecursiveGlob(rootdir, "*.yaml") + yamls = append(yamls, pathlib.RecursiveGlob(rootdir, "*.yml")...) + diagnoseFilesUnmarshal(yaml.Unmarshal, "YAML", rootdir, yamls, target) +} + +func addRobotDiagnostics(robotfile string, target *common.DiagnosticStatus, production bool) { + supportGeneralUrl := settings.Global.DocsLink("troubleshooting") + config, err := robot.LoadRobotYaml(robotfile, false) + diagnose := target.Diagnose("Robot") + if err != nil { + diagnose.Fail(0, supportGeneralUrl, "About robot.yaml: %v", err) + } else { + config.Diagnostics(target, production) + } + addFileDiagnostics(filepath.Dir(robotfile), target) +} + +func RunRobotDiagnostics(robotfile string, production bool) *common.DiagnosticStatus { + result := &common.DiagnosticStatus{ + Details: make(map[string]string), + Checks: []*common.DiagnosticCheck{}, + } + addRobotDiagnostics(robotfile, result, production) + return result +} + +func PrintRobotDiagnostics(robotfile string, json, production bool) error { + result := RunRobotDiagnostics(robotfile, production) + if json { + jsonDiagnostics(os.Stdout, result) + } else { + humaneDiagnostics(os.Stderr, result, true) + } + return nil +} diff --git a/operations/security.go b/operations/encryptionv1.go similarity index 85% rename from operations/security.go rename to operations/encryptionv1.go index ab2340a8..92c5305d 100644 --- a/operations/security.go +++ b/operations/encryptionv1.go @@ -14,8 +14,15 @@ import ( "errors" "fmt" "io" + + "github.com/robocorp/rcc/common" ) +type Ephemeral interface { + RequestBody(interface{}) (io.Reader, error) + Decode([]byte) ([]byte, error) +} + type EncryptionKeys struct { Iv string `json:"iv"` Atag string `json:"atag"` @@ -35,22 +42,16 @@ func Decoded(content string) ([]byte, error) { return base64.StdEncoding.DecodeString(content) } -func GenerateEphemeralKey() (*EncryptionV1, error) { - key, err := rsa.GenerateKey(rand.Reader, 4096) +func GenerateEphemeralKey() (Ephemeral, error) { + common.Timeline("start ephemeral key generation") + defer common.Timeline("done ephemeral key generation") + key, err := rsa.GenerateKey(rand.Reader, 2048) if err != nil { return nil, err } return &EncryptionV1{key}, nil } -func (it *EncryptionV1) PublicDER() string { - public, ok := it.Public().(*rsa.PublicKey) - if !ok { - return "" - } - return base64.StdEncoding.EncodeToString(x509.MarshalPKCS1PublicKey(public)) -} - func (it *EncryptionV1) PublicPEM() string { public, ok := it.Public().(*rsa.PublicKey) if !ok { @@ -114,7 +115,7 @@ func (it *EncryptionV1) Decode(blob []byte) ([]byte, error) { return nil, err } if aesgcm.NonceSize() != len(iv) { - return nil, errors.New(fmt.Sprintf("Size difference in AES GCM nonce, %d vs. %d!", aesgcm.NonceSize(), len(iv))) + return nil, fmt.Errorf("Size difference in AES GCM nonce, %d vs. %d!", aesgcm.NonceSize(), len(iv)) } atag, err := Decoded(content.Encryption.Atag) if err != nil { diff --git a/operations/fixing.go b/operations/fixing.go index 43de9731..4b23fd10 100644 --- a/operations/fixing.go +++ b/operations/fixing.go @@ -2,7 +2,6 @@ package operations import ( "bytes" - "io/ioutil" "os" "path/filepath" "strings" @@ -17,10 +16,21 @@ var ( ) func init() { + nonExecutableExtensions[".svg"] = true + nonExecutableExtensions[".bmp"] = true + nonExecutableExtensions[".png"] = true + nonExecutableExtensions[".gif"] = true + nonExecutableExtensions[".jpg"] = true + nonExecutableExtensions[".jpeg"] = true nonExecutableExtensions[".md"] = true nonExecutableExtensions[".txt"] = true nonExecutableExtensions[".htm"] = true nonExecutableExtensions[".html"] = true + nonExecutableExtensions[".csv"] = true + nonExecutableExtensions[".yml"] = true + nonExecutableExtensions[".yaml"] = true + nonExecutableExtensions[".json"] = true + nonExecutableExtensions[".robot"] = true } func ToUnix(content []byte) []byte { @@ -29,12 +39,12 @@ func ToUnix(content []byte) []byte { } func fixShellFile(fullpath string) { - content, err := ioutil.ReadFile(fullpath) + content, err := os.ReadFile(fullpath) if err != nil || bytes.IndexByte(content, '\r') < 0 { return } common.Debug("Fixing newlines in file: %v", fullpath) - err = ioutil.WriteFile(fullpath, ToUnix(content), 0o755) + err = pathlib.WriteFile(fullpath, ToUnix(content), 0o755) if err != nil { common.Log("Failure %v while fixing newlines in %v!", err, fullpath) } @@ -43,7 +53,7 @@ func fixShellFile(fullpath string) { func makeExecutable(fullpath string, file os.FileInfo) { extension := strings.ToLower(filepath.Ext(file.Name())) ignore, ok := nonExecutableExtensions[extension] - if ok && ignore || file.Mode() == 0o755 { + if ok && ignore || file.Mode() == 0o755 || strings.HasPrefix(file.Name(), ".") { return } os.Chmod(fullpath, 0o755) @@ -75,15 +85,12 @@ func ensureFilesExecutable(dir string) { } func FixRobot(robotFile string) error { - config, err := robot.LoadYamlConfiguration(robotFile) + config, err := robot.LoadRobotYaml(robotFile, false) if err != nil { return err } - tasks := config.AvailableTasks() - for _, task := range tasks { - for _, path := range config.Paths(task) { - ensureFilesExecutable(path) - } + for _, path := range config.Paths() { + ensureFilesExecutable(path) } return nil } @@ -93,9 +100,5 @@ func FixDirectory(dir string) error { if pathlib.IsFile(primary) { return FixRobot(primary) } - secondary := filepath.Join(dir, "package.yaml") - if pathlib.IsFile(secondary) { - return FixRobot(secondary) - } return nil } diff --git a/operations/initialize.go b/operations/initialize.go index 347ddfe5..d18045c4 100644 --- a/operations/initialize.go +++ b/operations/initialize.go @@ -3,17 +3,158 @@ package operations import ( "archive/zip" "bytes" - "errors" "fmt" + "os" "path/filepath" "sort" "strings" "github.com/robocorp/rcc/blobs" + "github.com/robocorp/rcc/cloud" "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/fail" "github.com/robocorp/rcc/pathlib" + "github.com/robocorp/rcc/pretty" + "github.com/robocorp/rcc/settings" + "gopkg.in/yaml.v2" ) +type StringMap map[string]string +type StringPair [2]string +type StringPairList []StringPair + +type MetaTemplates struct { + Date string `yaml:"date"` + Hash string `yaml:"hash"` + Templates StringMap `yaml:"templates"` + Url string `yaml:"url"` +} + +func (it *MetaTemplates) MatchingHash(hash string) bool { + return strings.EqualFold(strings.TrimSpace(hash), strings.TrimSpace(it.Hash)) +} + +func (it StringPairList) Len() int { + return len(it) +} + +func (it StringPairList) Less(left, right int) bool { + return it[left][0] < it[right][0] +} + +func (it StringPairList) Swap(left, right int) { + it[left], it[right] = it[right], it[left] +} + +func parseTemplateInfo(raw []byte) (ingore *MetaTemplates, err error) { + var metadata MetaTemplates + err = yaml.Unmarshal(raw, &metadata) + if err != nil { + return nil, err + } + return &metadata, nil +} + +func TemplateInfo(filename string) (ingore *MetaTemplates, err error) { + defer fail.Around(&err) + + raw, err := os.ReadFile(filename) + fail.On(err != nil, "Failure reading %q, reason: %v", filename, err) + metadata, err := parseTemplateInfo(raw) + fail.On(err != nil, "Failure parsing %q, reason: %v", filename, err) + return metadata, nil +} + +func templatesYamlPart() string { + return filepath.Join(common.TemplateLocation(), "templates.yaml.part") +} + +func templatesYamlFinal() string { + return filepath.Join(common.TemplateLocation(), "templates.yaml") +} + +func templatesZipPart() string { + return filepath.Join(common.TemplateLocation(), "templates.zip.part") +} + +func TemplatesZip() string { + return filepath.Join(common.TemplateLocation(), "templates.zip") +} + +func needNewTemplates() (ignore *MetaTemplates, err error) { + defer fail.Around(&err) + + metadata := settings.Global.TemplatesYamlURL() + if len(metadata) == 0 { + common.Debug("No URL for templates.yaml available.") + return nil, nil + } + partfile := templatesYamlPart() + err = cloud.Download(metadata, partfile) + fail.On(err != nil, "Failure loading %q, reason: %s", metadata, err) + meta, err := TemplateInfo(partfile) + fail.On(err != nil, "%s", err) + fail.On(!strings.HasPrefix(meta.Url, "https:"), "Location for templates.zip is not https: %q", meta.Url) + hash, err := pathlib.Sha256(TemplatesZip()) + if err != nil || !meta.MatchingHash(hash) { + return meta, nil + } + return nil, nil +} + +func activeTemplateInfo(internal bool) (*MetaTemplates, error) { + if !internal { + meta, err := TemplateInfo(templatesYamlFinal()) + if err == nil { + return meta, nil + } + } + raw, err := blobs.Asset("assets/templates.yaml") + if err != nil { + return nil, err + } + return parseTemplateInfo(raw) +} + +func downloadTemplatesZip(meta *MetaTemplates) (err error) { + defer fail.Around(&err) + + partfile := templatesZipPart() + err = cloud.Download(meta.Url, partfile) + fail.On(err != nil, "Failure loading %q, reason: %s", meta.Url, err) + hash, err := pathlib.Sha256(partfile) + fail.On(err != nil, "Failure hashing %q, reason: %s", partfile, err) + fail.On(!meta.MatchingHash(hash), "Received broken templates.zip, hash mismatch from expected %q vs. actual %q", meta.Hash, hash) + return nil +} + +func ensureUpdatedTemplates() { + err := updateTemplates() + if err != nil { + pretty.Warning("Problem updating templates.zip, reason: %v", err) + } +} + +func updateTemplates() (err error) { + defer fail.Around(&err) + + defer os.Remove(templatesZipPart()) + defer os.Remove(templatesYamlPart()) + + meta, err := needNewTemplates() + fail.On(err != nil, "%s", err) + if meta == nil { + return nil + } + err = downloadTemplatesZip(meta) + fail.On(err != nil, "%s", err) + err = os.Rename(templatesYamlPart(), templatesYamlFinal()) + alt := os.Rename(templatesZipPart(), TemplatesZip()) + fail.On(alt != nil, "%s", alt) + fail.On(err != nil, "%s", err) + return nil +} + func unpack(content []byte, directory string) error { common.Debug("Initializing:") size := int64(len(content)) @@ -36,26 +177,61 @@ func unpack(content []byte, directory string) error { } common.Debug("Done.") if !success { - return errors.New(fmt.Sprintf("Problems while initializing robot. Use --debug to see details.")) + return fmt.Errorf("Problems while initializing robot. Use --debug to see details.") } return nil } -func ListTemplates() []string { - assets := blobs.AssetNames() - result := make([]string, 0, len(assets)) - for _, name := range blobs.AssetNames() { - if !strings.HasPrefix(name, "assets") || !strings.HasSuffix(name, ".zip") { - continue - } - result = append(result, strings.TrimSuffix(filepath.Base(name), filepath.Ext(name))) +func ListTemplatesWithDescription(internal bool) StringPairList { + ensureUpdatedTemplates() + result := make(StringPairList, 0, 10) + meta, err := activeTemplateInfo(internal) + if err != nil { + pretty.Warning("Problem getting template list, reason: %v", err) + return result + } + for name, description := range meta.Templates { + result = append(result, StringPair{name, description}) + } + sort.Sort(result) + return result +} + +func ListTemplates(internal bool) []string { + ensureUpdatedTemplates() + pairs := ListTemplatesWithDescription(internal) + result := make([]string, 0, len(pairs)) + for _, pair := range pairs { + result = append(result, pair[0]) } - sort.Strings(result) return result } -func InitializeWorkarea(directory, name string, force bool) error { - content, err := blobs.Asset(fmt.Sprintf("assets/%s.zip", name)) +func templateByName(name string, internal bool) ([]byte, error) { + zipfile := TemplatesZip() + blobname := fmt.Sprintf("assets/%s.zip", name) + if internal || !pathlib.IsFile(zipfile) { + return blobs.Asset(blobname) + } + unzipper, err := newUnzipper(zipfile, false) + if err != nil { + return nil, err + } + defer unzipper.Close() + zipname := fmt.Sprintf("%s.zip", name) + blob, err := unzipper.Asset(zipname) + if err != nil { + return nil, err + } + if blob != nil { + return blob, nil + } + return blobs.Asset(blobname) +} + +func InitializeWorkarea(directory, name string, internal, force bool) error { + ensureUpdatedTemplates() + content, err := templateByName(name, internal) if err != nil { return err } @@ -71,6 +247,5 @@ func InitializeWorkarea(directory, name string, force bool) error { if err != nil { return err } - UpdateRobot(fullpath) return unpack(content, fullpath) } diff --git a/operations/issues.go b/operations/issues.go new file mode 100644 index 00000000..39bf80e9 --- /dev/null +++ b/operations/issues.go @@ -0,0 +1,164 @@ +package operations + +import ( + "bytes" + "fmt" + "os" + "path/filepath" + + "github.com/robocorp/rcc/cloud" + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/pathlib" + "github.com/robocorp/rcc/settings" + "github.com/robocorp/rcc/xviper" +) + +const ( + issueUrl = `/diagnostics-v1/issue` +) + +func loadToken(reportFile string) (Token, error) { + content, err := os.ReadFile(reportFile) + if err != nil { + return nil, err + } + token := make(Token) + err = token.FromJson(content) + if err != nil { + return nil, err + } + return token, nil +} + +func createIssueZip(attachmentsFiles []string) (string, error) { + zipfile := filepath.Join(common.ProductTemp(), "attachments.zip") + zipper, err := newZipper(zipfile) + if err != nil { + return "", err + } + defer zipper.Close() + for index, attachment := range attachmentsFiles { + niceName := fmt.Sprintf("%x_%s", index+1, filepath.Base(attachment)) + zipper.Add(attachment, niceName, nil) + } + // getting settings.yaml is optional, it should not break issue reporting + config, err := settings.SummonSettings() + if err != nil { + return zipfile, nil + } + blob, err := config.AsYaml() + if err != nil { + return zipfile, nil + } + niceName := fmt.Sprintf("%x_settings.yaml", len(attachmentsFiles)+1) + zipper.AddBlob(niceName, blob) + return zipfile, nil +} + +func createDiagnosticsReport(robotfile string) (string, *common.DiagnosticStatus, error) { + file := filepath.Join(common.ProductTemp(), "diagnostics.txt") + diagnostics, err := ProduceDiagnostics(file, robotfile, false, false, false) + if err != nil { + return "", nil, err + } + return file, diagnostics, nil +} + +func virtualName(filename string) (string, error) { + digest, err := pathlib.Sha256(filename) + if err != nil { + return "", err + } + return fmt.Sprintf("attachments_%s.zip", digest[:16]), nil +} + +func ReportIssue(email, robotFile, reportFile string, attachmentsFiles []string, dryrun bool) error { + issueHost := settings.Global.IssuesURL() + if len(issueHost) == 0 { + return nil + } + cloud.InternalBackgroundMetric(common.ControllerIdentity(), "rcc.submit.issue", common.Version) + token, err := loadToken(reportFile) + if err != nil { + return err + } + diagnostics, data, err := createDiagnosticsReport(robotFile) + if err == nil { + attachmentsFiles = append(attachmentsFiles, diagnostics) + } + plan, ok := data.Details["robot-conda-plan"] + if ok { + attachmentsFiles = append(attachmentsFiles, plan) + } + attachmentsFiles = append(attachmentsFiles, reportFile) + filename, err := createIssueZip(attachmentsFiles) + if err != nil { + return err + } + shortname, err := virtualName(filename) + if err != nil { + return err + } + installationId := xviper.TrackingIdentity() + token["installationId"] = installationId + token["account-email"] = email + token["fileName"] = shortname + token["controller"] = common.ControllerIdentity() + _, ok = token["platform"] + if !ok { + token["platform"] = common.Platform() + } + issueReport, err := token.AsJson() + if err != nil { + return err + } + if dryrun { + metaForm := make(Token) + metaForm["report"] = token + metaForm["zipfile"] = filename + report, err := metaForm.AsJson() + if err != nil { + return err + } + fmt.Fprintln(os.Stdout, report) + return nil + } + common.Trace(issueReport) + client, err := cloud.NewClient(issueHost) + if err != nil { + return err + } + request := client.NewRequest(issueUrl) + request.Headers[contentType] = applicationJson + request.Body = bytes.NewBuffer([]byte(issueReport)) + response := client.Post(request) + json := make(Token) + err = json.FromJson(response.Body) + if err != nil { + return err + } + postInfo, ok := json["attachmentPostInfo"].(map[string]interface{}) + if !ok { + return fmt.Errorf("Could not get attachmentPostInfo!") + } + url, ok := postInfo["url"].(string) + if !ok { + return fmt.Errorf("Could not get URL from attachmentPostInfo!") + } + fields, ok := postInfo["fields"].(map[string]interface{}) + if !ok { + return fmt.Errorf("Could not get fields from attachmentPostInfo!") + } + return MultipartUpload(url, toStringMap(fields), shortname, filename) +} + +func toStringMap(entries map[string]interface{}) map[string]string { + result := make(map[string]string) + for key, value := range entries { + text, ok := value.(string) + if ok { + result[key] = text + } + } + return result +} diff --git a/operations/metrics.go b/operations/metrics.go deleted file mode 100644 index 0f0074b6..00000000 --- a/operations/metrics.go +++ /dev/null @@ -1,39 +0,0 @@ -package operations - -import ( - "fmt" - "net/url" - "time" - - "github.com/robocorp/rcc/cloud" - "github.com/robocorp/rcc/common" - "github.com/robocorp/rcc/xviper" -) - -const ( - trackingUrl = `/metric-v1/%v/%v/%v/%v/%v` - metricsHost = `https://telemetry.robocorp.com` -) - -func sendMetric(kind, name, value string) { - client, err := cloud.NewClient(metricsHost) - if err != nil { - common.Debug("ERROR: %v", err) - return - } - timestamp := time.Now().UnixNano() - url := fmt.Sprintf(trackingUrl, url.PathEscape(kind), timestamp, url.PathEscape(xviper.TrackingIdentity()), url.PathEscape(name), url.PathEscape(value)) - common.Debug("DEBUG: Sending metric as %v%v", metricsHost, url) - client.Put(client.NewRequest(url)) -} - -func SendMetric(kind, name, value string) { - common.Debug("DEBUG: SendMetric kind:%v name:%v value:%v send:%v", kind, name, value, xviper.CanTrack()) - if xviper.CanTrack() { - sendMetric(kind, name, value) - } -} - -func BackgroundMetric(kind, name, value string) { - go SendMetric(kind, name, value) -} diff --git a/operations/netdiagnostics.go b/operations/netdiagnostics.go new file mode 100644 index 00000000..86856744 --- /dev/null +++ b/operations/netdiagnostics.go @@ -0,0 +1,182 @@ +package operations + +import ( + "crypto/sha256" + "fmt" + "net/url" + "strings" + + "github.com/robocorp/rcc/cloud" + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/fail" + "github.com/robocorp/rcc/set" + "github.com/robocorp/rcc/settings" + "gopkg.in/yaml.v2" +) + +type ( + WebConfig struct { + URL string `yaml:"url"` + Codes []int `yaml:"codes"` + Fingerprint string `yaml:"content-sha256,omitempty"` + } + + NetConfig struct { + DNS []string `yaml:"dns-lookup"` + TLS []string `yaml:"tls-verify"` + Head []*WebConfig `yaml:"head-request"` + Get []*WebConfig `yaml:"get-request"` + } + + Configuration struct { + Network *NetConfig `yaml:"network"` + } + + webtool func(string) (int, string, error) +) + +func (it *NetConfig) Hostnames() []string { + result := make([]string, 0, len(it.DNS)+len(it.TLS)) + result = append(result, it.DNS...) + result = append(result, it.TLS...) + for _, entry := range it.Head { + parsed, err := url.Parse(entry.URL) + if err == nil { + result = append(result, parsed.Hostname()) + } + } + for _, entry := range it.Get { + parsed, err := url.Parse(entry.URL) + if err == nil { + result = append(result, parsed.Hostname()) + } + } + return set.Set(result) +} + +func parseNetworkDiagnosticConfig(body []byte) (*Configuration, error) { + config := &Configuration{} + err := yaml.Unmarshal(body, config) + if err != nil { + return nil, err + } + return config, nil +} + +func networkDiagnostics(config *Configuration, target *common.DiagnosticStatus) []*common.DiagnosticCheck { + supportUrl := settings.Global.DocsLink("troubleshooting/firewall-and-proxies") + if config == nil || config.Network == nil { + return target.Checks + } + diagnosticsStopwatch := common.Stopwatch("Full network diagnostics time was about") + hostnames := config.Network.Hostnames() + dnsStopwatch := common.Stopwatch("DNS lookup time for %d hostnames was about", len(hostnames)) + for _, host := range hostnames { + target.Checks = append(target.Checks, dnsLookupCheck(host)) + } + target.Details["dns-lookup-time"] = dnsStopwatch.Text() + tlsRoots := make(map[string]bool) + tlsStopwatch := common.Stopwatch("TLS verification time for %d hostnames was about", len(hostnames)) + for _, host := range hostnames { + target.Checks = append(target.Checks, tlsCheckHost(host, tlsRoots)...) + } + target.Details["tls-lookup-time"] = tlsStopwatch.Text() + if len(hostnames) > 1 && len(tlsRoots) == 1 { + for name, _ := range tlsRoots { + target.Details["tls-proxy-firewall"] = name + } + } else { + target.Details["tls-proxy-firewall"] = "undetectable" + } + headStopwatch := common.Stopwatch("HEAD request time for %d requests was about", len(config.Network.Head)) + for _, entry := range config.Network.Head { + target.Checks = append(target.Checks, webDiagnostics("HEAD", common.CategoryNetworkHEAD, headRequest, entry, supportUrl)...) + } + target.Details["head-time"] = headStopwatch.Text() + getStopwatch := common.Stopwatch("GET request time for %d requests was about", len(config.Network.Get)) + for _, entry := range config.Network.Get { + target.Checks = append(target.Checks, webDiagnostics("GET", common.CategoryNetworkCanary, getRequest, entry, supportUrl)...) + } + target.Details["get-time"] = getStopwatch.Text() + target.Details["diagnostics-time"] = diagnosticsStopwatch.Text() + return target.Checks +} + +func webDiagnostics(label string, category uint64, tool webtool, item *WebConfig, supportUrl string) []*common.DiagnosticCheck { + result := make([]*common.DiagnosticCheck, 0, 2) + code, fingerprint, err := tool(item.URL) + valid := set.Set(item.Codes) + member := set.Member(valid, code) + if member { + result = append(result, &common.DiagnosticCheck{ + Type: "network", + Category: category, + Status: statusOk, + Message: fmt.Sprintf("%s %q successful with status %d.", label, item.URL, code), + Link: supportUrl, + }) + } else { + result = append(result, &common.DiagnosticCheck{ + Type: "network", + Category: category, + Status: statusFail, + Message: fmt.Sprintf("%s %q failed with status %d.", label, item.URL, code), + Link: supportUrl, + }) + } + if err != nil { + result = append(result, &common.DiagnosticCheck{ + Type: "network", + Category: category, + Status: statusWarning, + Message: fmt.Sprintf("%s %q resulted error: %v.", label, item.URL, err), + Link: supportUrl, + }) + } + if len(item.Fingerprint) > 0 && !strings.HasPrefix(fingerprint, item.Fingerprint) { + result = append(result, &common.DiagnosticCheck{ + Type: "network", + Category: category, + Status: statusWarning, + Message: fmt.Sprintf("%s %q fingerprint mismatch: expected %q, but got %q instead.", label, item.URL, item.Fingerprint, fingerprint), + Link: supportUrl, + }) + } + return result +} + +func digest(body []byte) string { + digester := sha256.New() + digester.Write(body) + return fmt.Sprintf("%02x", digester.Sum([]byte{})) +} + +func headRequest(link string) (code int, fingerprint string, err error) { + defer fail.Around(&err) + + client, err := cloud.NewClient(link) + fail.On(err != nil, "Client for %q failed, reason: %v", link, err) + if common.TraceFlag() { + client = client.WithTracing() + } + request := client.NewRequest("") + response := client.Head(request) + fail.On(response.Err != nil, "HEAD request to %q failed, reason: %v", link, response.Err) + + return response.Status, digest(response.Body), nil +} + +func getRequest(link string) (code int, fingerprint string, err error) { + defer fail.Around(&err) + + client, err := cloud.NewClient(link) + fail.On(err != nil, "Client for %q failed, reason: %v", link, err) + if common.TraceFlag() { + client = client.WithTracing() + } + request := client.NewRequest("") + response := client.Get(request) + fail.On(response.Err != nil, "HEAD request to %q failed, reason: %v", link, response.Err) + + return response.Status, digest(response.Body), nil +} diff --git a/operations/processtree.go b/operations/processtree.go new file mode 100644 index 00000000..6bc4b1e5 --- /dev/null +++ b/operations/processtree.go @@ -0,0 +1,258 @@ +package operations + +import ( + "fmt" + "os" + "time" + + "github.com/mitchellh/go-ps" + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/pretty" + "github.com/robocorp/rcc/set" +) + +type ( + ChildMap map[int]string + ProcessMap map[int]*ProcessNode + ProcessNodes []*ProcessNode + ProcessNode struct { + Pid int + Parent int + White bool + Executable string + Children ProcessMap + } +) + +var ( + processBlacklist = make(map[int]int) +) + +func init() { + processes, err := ProcessMapNow() + if err == nil { + rcc := os.Getpid() + for _, process := range processes { + if process.Pid != rcc { + processBlacklist[process.Pid] = process.Parent + } + } + } +} + +func NewProcessNode(core ps.Process) *ProcessNode { + return &ProcessNode{ + Pid: core.Pid(), + Parent: core.PPid(), + White: true, + Executable: core.Executable(), + Children: make(ProcessMap), + } +} + +func ProcessMapNow() (ProcessMap, error) { + processes, err := ps.Processes() + if err != nil { + return nil, err + } + result := make(ProcessMap) + for _, process := range processes { + node := NewProcessNode(process) + old, ok := processBlacklist[node.Pid] + if ok && old == node.Parent { + continue + } + node.White = !ok + result[node.Pid] = node + } + for pid, process := range result { + parent, ok := result[process.Parent] + if ok { + parent.Children[pid] = process + } + } + return result, nil +} + +func (it ProcessMap) Keys() []int { + return set.Keys(it) +} + +func (it ProcessMap) Roots() []int { + roots := []int{} + for candidate, node := range it { + _, ok := it[node.Parent] + if !ok { + roots = append(roots, candidate) + } + } + return set.Sort(roots) +} + +func (it *ProcessNode) warnings(additional ProcessMap) { + if len(it.Children) > 0 { + pretty.Warning("%q process %d (parent %d) still has running subprocesses:", it.Executable, it.Pid, it.Parent) + it.warningTree("> ", false, 20) + } else { + pretty.Warning("%q process %d (parent %d) still has running migrated processes:", it.Executable, it.Pid, it.Parent) + } + if len(additional) > 0 { + pretty.Warning("+ migrated process still running:") + for _, key := range additional.Roots() { + additional[key].warningTree("| ", true, 20) + } + } + pretty.Note("Depending on OS, above processes may prevent robot to close properly.") + pretty.Note("Few reasons why this might be happening are:") + pretty.Note("- robot is not properly releasing all resources that it is using") + pretty.Note("- robot is generating background processes that don't complete before robot tries to exit") + pretty.Note("- there was failure inside robot, which caused robot to exit without proper cleanup") + pretty.Note("- developer intentionally left processes running, which is not good for repeatable automation") + pretty.Highlight("So if you see this message, and robot still seems to be running, it is not!") + pretty.Highlight("You now have to take action and stop those processes that are preventing robot to complete.") + pretty.Highlight("Example cleanup command: %s", common.GenerateKillCommand(additional.Keys())) +} + +func (it *ProcessNode) warningTree(prefix string, newparent bool, limit int) { + kind := "leaf" + if len(it.Children) > 0 { + kind = "container" + } + var grey string + if !it.White { + grey = " (grey listed)" + } + if newparent { + kind = fmt.Sprintf("%s -> new parent PID: #%d", kind, it.Parent) + } else { + kind = fmt.Sprintf("%s under #%d", kind, it.Parent) + } + pretty.Warning("%s#%d %q <%s>%s%s", prefix, it.Pid, it.Executable, kind, pretty.Grey, grey) + common.RunJournal("orphan process", fmt.Sprintf("parent=%d pid=%d name=%s greylist=%v", it.Parent, it.Pid, it.Executable, !it.White), "process pollution") + if limit < 0 { + pretty.Warning("%s Maximum recursion depth detected. Truncating output here.", prefix) + return + } + indent := prefix + "| " + for _, key := range it.Children.Keys() { + it.Children[key].warningTree(indent, false, limit-1) + } +} + +func SubprocessWarning(seen ChildMap, use bool) error { + before := len(seen) + if before == 0 { + common.Debug("No tracked subprocesses, which is a good thing.") + return nil + } + time.Sleep(1 * time.Second) // small nap to let things settle before asking all processes + processes, err := ProcessMapNow() + if err != nil { + return err + } + removeStaleChildren(processes, seen) + after := len(seen) + pretty.DebugNote("Final subprocess count %d -> %d. %v", before, after, seen) + if after == 0 { + common.Debug("No active tracked subprocesses anymore, and that is a good thing.") + return nil + } + self, ok := processes[os.Getpid()] + if !ok { + return fmt.Errorf("For some reason, could not find own process in process map.") + } + additional := make(ProcessMap) + for pid, executable := range seen { + ref, ok := processes[pid] + if ok && executable == ref.Executable { + additional[pid] = ref + } + } + if len(self.Children) > 0 || len(additional) > 0 { + self.warnings(additional) + } + return nil +} + +func removeStaleChildren(processes ProcessMap, seen ChildMap) bool { + removed := false + for key, name := range seen { + found, ok := processes[key] + if !ok || found.Executable != name { + delete(seen, key) + removed = true + } + } + return removed +} + +func updateActiveChildrenLoop(start *ProcessNode, seen ChildMap) bool { + updated := false + counted := make(map[int]bool) + counted[start.Pid] = true + at, todo := 0, ProcessNodes{start} + for at < len(todo) { + for pid, child := range todo[at].Children { + if counted[pid] { + continue + } + counted[pid] = true + _, previously := seen[pid] + seen[pid] = child.Executable + todo = append(todo, child) + if !previously { + updated = true + } + } + at += 1 + } + return updated +} + +func updateSeenChildren(pid int, processes ProcessMap, seen ChildMap) bool { + source, ok := processes[pid] + if ok { + removed := removeStaleChildren(processes, seen) + updated := updateActiveChildrenLoop(source, seen) + return removed || updated + } + return false +} + +func WatchChildren(pid int, delay time.Duration) chan ChildMap { + common.Debug("Process blacklist size is %d processes.", len(processBlacklist)) + pipe := make(chan ChildMap) + go babySitter(pid, pipe, delay) + return pipe +} + +func babySitter(pid int, reply chan ChildMap, delay time.Duration) { + defer close(reply) + seen := make(ChildMap) + failures, broadcasted := 0, 0 + defer common.RunJournal("processes", "final", "count: %d", broadcasted) +forever: + for failures < 10 { + updated := false + processes, err := ProcessMapNow() + if err == nil { + updated = updateSeenChildren(pid, processes, seen) + failures = 0 + } else { + common.Debug("Process snapshot failure: %v", err) + } + if updated { + active := len(seen) + pretty.DebugNote("Active subprocess count %d -> %d. %v", broadcasted, active, seen) + common.RunJournal("processes", "updated", "count from %d to %d ... %v", broadcasted, active, seen) + broadcasted = active + } + select { + case reply <- seen: + break forever + case <-time.After(delay): + continue forever + } + } + common.Debug("Final active subprocess count was %d.", broadcasted) +} diff --git a/operations/pull.go b/operations/pull.go new file mode 100644 index 00000000..97656895 --- /dev/null +++ b/operations/pull.go @@ -0,0 +1,166 @@ +package operations + +import ( + "bufio" + "bytes" + "crypto/sha256" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" + + "github.com/robocorp/rcc/cloud" + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/fail" + "github.com/robocorp/rcc/htfs" + "github.com/robocorp/rcc/pathlib" + "github.com/robocorp/rcc/settings" + "github.com/robocorp/rcc/xviper" +) + +const ( + X_RCC_RANDOM_IDENTITY = `X-Rcc-Random-Identity` + AUTHORIZATION = "Authorization" +) + +func pullOriginFingerprints(origin, catalogName string) (fingerprints string, count int, err error) { + defer fail.Around(&err) + + common.TimelineBegin("pull rccremote origin fingerprints") + defer common.TimelineEnd() + + client, err := cloud.NewUnsafeClient(origin) + fail.On(err != nil, "Could not create web client for %q, reason: %v", origin, err) + + url := fmt.Sprintf("%s/parts/%s", origin, catalogName) + request := client.NewRequest(fmt.Sprintf("/parts/%s", catalogName)) + request.Headers[X_RCC_RANDOM_IDENTITY] = common.RandomIdentifier() + authorization, ok := common.RccRemoteAuthorization() + if ok { + request.Headers[AUTHORIZATION] = authorization + } + response := client.Get(request) + common.Timeline("status %d from GET %q", response.Status, url) + + fail.On(response.Status != 200, "Problem with parts request, status=%d, body=%s", response.Status, response.Body) + + stream := bufio.NewReader(bytes.NewReader(response.Body)) + collection := make([]string, 0, 2048) + for { + line, err := stream.ReadString('\n') + flat := strings.TrimSpace(line) + if len(flat) > 0 { + fullpath := htfs.ExactDefaultLocation(flat) + if !pathlib.IsFile(fullpath) { + collection = append(collection, flat) + } + } + if err == io.EOF { + common.Timeline("total of %d parts in catalog %q", len(collection), catalogName) + return strings.Join(collection, "\n"), len(collection), nil + } + fail.On(err != nil, "STREAM error: %v", err) + } + + return "", 0, fmt.Errorf("Unexpected reach of code that should never happen.") +} + +func downloadMissingEnvironmentParts(count int, origin, catalogName, selection string) (filename string, err error) { + defer fail.Around(&err) + + common.TimelineBegin("download %d parts + catalog from %q", count, origin) + defer common.TimelineEnd() + + url := fmt.Sprintf("%s/delta/%s", origin, catalogName) + + body := strings.NewReader(selection) + filename = filepath.Join(pathlib.TempDir(), fmt.Sprintf("rccremote_%x.zip", os.Getpid())) + + client := &http.Client{Transport: settings.Global.ConfiguredHttpTransport()} + request, err := http.NewRequest("POST", url, body) + fail.On(err != nil, "Failed create request to %q failed, reason: %v", url, err) + + request.Header.Add("robocorp-installation-id", xviper.TrackingIdentity()) + request.Header.Add("User-Agent", common.UserAgent()) + request.Header.Add(X_RCC_RANDOM_IDENTITY, common.RandomIdentifier()) + authorization, ok := common.RccRemoteAuthorization() + if ok { + request.Header.Add(AUTHORIZATION, authorization) + } + + response, err := client.Do(request) + fail.On(err != nil, "Web request to %q failed, reason: %v", url, err) + defer response.Body.Close() + + common.Timeline("status %d from POST %q", response.StatusCode, url) + + fail.On(response.StatusCode < 200 || 299 < response.StatusCode, "%s (%s)", response.Status, url) + + out, err := pathlib.Create(filename) + fail.On(err != nil, "Creating temporary file %q failed, reason: %v", filename, err) + defer pathlib.TryRemove("temporary", filename) + + digest := sha256.New() + many := io.MultiWriter(out, digest) + + common.Debug("Downloading %s <%s> -> %s", url, response.Status, filename) + + _, err = io.Copy(many, response.Body) + fail.On(err != nil, "Download failed, reason: %v", err) + + err = out.Sync() + fail.On(err != nil, "Sync of %q failed, reason: %v", filename, err) + + err = out.Close() + fail.On(err != nil, "Closing %q failed, reason: %v", filename, err) + + sum := fmt.Sprintf("%02x", digest.Sum(nil)) + finalname := filepath.Join(pathlib.TempDir(), fmt.Sprintf("rccremote_%s.zip", sum)) + err = pathlib.TryRename("delta", filename, finalname) + fail.On(err != nil, "Rename %q -> %q failed, reason: %v", filename, finalname, err) + + return finalname, nil +} + +func ProtectedImport(filename string) (err error) { + defer fail.Around(&err) + + lockfile := common.HolotreeLock() + completed := pathlib.LockWaitMessage(lockfile, "Serialized environment import [holotree lock]") + locker, err := pathlib.Locker(lockfile, 30000, common.SharedHolotree) + completed() + fail.On(err != nil, "Could not get lock for holotree. Quiting.") + defer locker.Release() + + common.Timeline("Import %v", filename) + return Unzip(common.HololibLocation(), filename, true, false, false) +} + +func PullCatalog(origin, catalogName string, useLock bool) (err error) { + defer fail.Around(&err) + + common.TimelineBegin("hololib+catalog pull start") + defer common.TimelineEnd() + + common.Timeline("pulling %q parts from %q", catalogName, origin) + + unknownSelected, count, err := pullOriginFingerprints(origin, catalogName) + fail.On(err != nil, "%v", err) + + filename, err := downloadMissingEnvironmentParts(count, origin, catalogName, unknownSelected) + fail.On(err != nil, "%v", err) + + common.Debug("Temporary content based filename is: %q", filename) + defer pathlib.TryRemove("temporary", filename) + + if useLock { + err = ProtectedImport(filename) + } else { + err = Unzip(common.HololibLocation(), filename, true, false, false) + } + fail.On(err != nil, "Failed to unzip %v to hololib, reason: %v", filename, err) + + return nil +} diff --git a/operations/rccversioncheck.go b/operations/rccversioncheck.go new file mode 100644 index 00000000..b0547dae --- /dev/null +++ b/operations/rccversioncheck.go @@ -0,0 +1,107 @@ +package operations + +import ( + "encoding/json" + "os" + "path/filepath" + + "github.com/robocorp/rcc/cloud" + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/conda" + "github.com/robocorp/rcc/fail" + "github.com/robocorp/rcc/pretty" + "github.com/robocorp/rcc/settings" +) + +type ( + rccVersions struct { + Tested []*versionInfo `json:"tested"` + } + versionInfo struct { + Version string `json:"version"` + When string `json:"when"` + } +) + +func rccReleaseInfoURL() string { + // https://downloads.robocorp.com/rcc/releases/index.json + return settings.Global.DownloadsLink("/rcc/releases/index.json") +} + +func rccVersionsJsonPart() string { + return filepath.Join(common.TemplateLocation(), "rcc.json.part") +} + +func rccVersionsJson() string { + return filepath.Join(common.TemplateLocation(), "rcc.json") +} + +func updateRccVersionInfo() (err error) { + defer fail.Around(&err) + + if !needNewRccInfo() { + return nil + } + return downloadVersionsJson() +} + +func needNewRccInfo() bool { + stat, err := os.Stat(rccVersionsJson()) + return err != nil || common.DayCountSince(stat.ModTime()) > 2 +} + +func downloadVersionsJson() (err error) { + defer fail.Around(&err) + + sourceURL := rccReleaseInfoURL() + partfile := rccVersionsJsonPart() + err = cloud.Download(sourceURL, partfile) + fail.On(err != nil, "Failure loading %q, reason: %s", sourceURL, err) + finaltarget := rccVersionsJson() + os.Remove(finaltarget) + return os.Rename(partfile, finaltarget) +} + +func loadVersionsInfo() (versions *rccVersions, err error) { + defer fail.Around(&err) + + blob, err := os.ReadFile(rccVersionsJson()) + fail.Fast(err) + versions = &rccVersions{} + err = json.Unmarshal(blob, versions) + fail.Fast(err) + return versions, nil +} + +func pickLatestTestedVersion(versions *rccVersions) (uint64, string, string) { + highest, text, when := uint64(0), "unknown", "unkown" + for _, version := range versions.Tested { + number, _ := conda.AsVersion(version.Version) + if number > highest { + text = version.Version + when = version.When + highest = number + } + } + return highest, text, when +} + +func RccVersionCheck() func() { + if common.IsBundled() { + common.Debug("Did not check newer version existence, since this is bundled case.") + return nil + } + updateRccVersionInfo() + versions, err := loadVersionsInfo() + if err != nil || versions == nil { + return nil + } + tested, textual, when := pickLatestTestedVersion(versions) + current, _ := conda.AsVersion(common.Version) + if tested == 0 || current == 0 || current >= tested { + return nil + } + return func() { + pretty.Note("Now running rcc %s. There is newer rcc version %s available, released at %s.", common.Version, textual, when) + } +} diff --git a/operations/robotcache.go b/operations/robotcache.go index bcadf263..58b8cabf 100644 --- a/operations/robotcache.go +++ b/operations/robotcache.go @@ -1,15 +1,14 @@ package operations import ( - "errors" "fmt" "os" "path/filepath" "regexp" + "runtime" "time" "github.com/robocorp/rcc/common" - "github.com/robocorp/rcc/conda" "github.com/robocorp/rcc/pathlib" ) @@ -26,6 +25,7 @@ func CacheRobot(filename string) error { if err != nil { return err } + common.Timeline("caching robot: %s -> %s", filename, digest) common.Debug("Digest for %v is %v.", fullpath, digest) _, exists := LookupRobot(digest) if exists { @@ -43,14 +43,15 @@ func CacheRobot(filename string) error { } if verify != digest { defer os.Remove(target) - return errors.New(fmt.Sprintf("Could not cache %v, reason: digest mismatch.", fullpath)) + return fmt.Errorf("Could not cache %v, reason: digest mismatch.", fullpath) } go CleanupOldestRobot() + runtime.Gosched() return nil } func cacheRobotFilename(digest string) string { - return filepath.Join(conda.RobotCache(), digest+".zip") + return filepath.Join(common.RobotCache(), digest+".zip") } func LookupRobot(digest string) (string, bool) { @@ -73,7 +74,7 @@ func CleanupOldestRobot() { func OldestRobot() (string, time.Time) { oldest, stamp := "", time.Now() deadline := time.Now().Add(-35 * 24 * time.Hour) - pathlib.Walk(conda.RobotCache(), pathlib.IgnoreNewer(deadline).Ignore, func(full, relative string, details os.FileInfo) { + pathlib.Walk(common.RobotCache(), pathlib.IgnoreNewer(deadline).Ignore, func(full, relative string, details os.FileInfo) { if zipPattern.MatchString(details.Name()) && details.ModTime().Before(stamp) { oldest, stamp = full, details.ModTime() } diff --git a/operations/robots.go b/operations/robots.go deleted file mode 100644 index 10bebc6e..00000000 --- a/operations/robots.go +++ /dev/null @@ -1,85 +0,0 @@ -package operations - -import ( - "os" - "path/filepath" - "sort" - "time" -) - -func UpdateRobot(directory string) error { - fullpath, err := filepath.Abs(directory) - if err != nil { - return err - } - cache, err := SummonCache() - if err != nil { - return err - } - defer cache.Save() - now := time.Now().Unix() - robot, ok := cache.Robots[fullpath] - if !ok { - robot = &Folder{ - Path: fullpath, - Created: now, - Updated: now, - Deleted: 0, - } - cache.Robots[fullpath] = robot - } - stat, err := os.Stat(fullpath) - if err != nil || !stat.IsDir() { - robot.Deleted = now - } - robot.Updated = now - return nil -} - -func sorted(folders []*Folder) { - sort.SliceStable(folders, func(left, right int) bool { - if folders[left].Deleted != folders[right].Deleted { - return folders[left].Deleted < folders[right].Deleted - } - return folders[left].Updated > folders[right].Updated - }) -} - -func detectDeadRobots() bool { - cache, err := SummonCache() - if err != nil { - return false - } - now := time.Now().Unix() - changed := false - for _, robot := range cache.Robots { - stat, err := os.Stat(robot.Path) - if err != nil || !stat.IsDir() { - robot.Deleted = now - changed = true - continue - } - if robot.Deleted > 0 && stat.IsDir() { - robot.Deleted = 0 - changed = true - } - } - if changed { - cache.Save() - } - return changed -} - -func ListRobots() ([]*Folder, error) { - detectDeadRobots() - cache, err := SummonCache() - if err != nil { - return nil, err - } - result := make([]*Folder, 0, len(cache.Robots)) - for _, value := range cache.Robots { - result = append(result, value) - } - sorted(result) - return result, nil -} diff --git a/operations/running.go b/operations/running.go index 0c4504f5..413ba80b 100644 --- a/operations/running.go +++ b/operations/running.go @@ -2,50 +2,162 @@ package operations import ( "fmt" + "os" "path/filepath" + "runtime" + "strings" + "time" + "github.com/robocorp/rcc/cloud" "github.com/robocorp/rcc/common" "github.com/robocorp/rcc/conda" + "github.com/robocorp/rcc/htfs" + "github.com/robocorp/rcc/journal" "github.com/robocorp/rcc/pathlib" "github.com/robocorp/rcc/pretty" "github.com/robocorp/rcc/robot" "github.com/robocorp/rcc/shell" ) +const ( + actualRun = `actual main robot run` + preRun = `pre-run script execution` + newEnvironment = `environment creation` +) + var ( rcHosts = []string{"RC_API_SECRET_HOST", "RC_API_WORKITEM_HOST"} rcTokens = []string{"RC_API_SECRET_TOKEN", "RC_API_WORKITEM_TOKEN"} ) +type TokenPeriod struct { + ValidityTime int // minutes + GracePeriod int // minutes +} + type RunFlags struct { + *TokenPeriod AccountName string WorkspaceId string - ValidityTime int EnvironmentFile string RobotYaml string + Assistant bool + NoPipFreeze bool } -func PipFreeze(searchPath pathlib.PathParts, directory, outputDir string, environment []string) bool { - pip, ok := searchPath.Which("pip", conda.FileExtensions) - if !ok { - return false +func (it *TokenPeriod) EnforceGracePeriod() *TokenPeriod { + if it == nil { + return it } - fullPip, err := filepath.EvalSymlinks(pip) + if it.GracePeriod < 5 { + it.GracePeriod = 5 + } + if it.GracePeriod > 120 { + it.GracePeriod = 120 + } + if it.ValidityTime < 15 { + it.ValidityTime = 15 + } + return it +} + +func asSeconds(minutes int) int { + return 60 * minutes +} + +func DefaultTokenPeriod() *TokenPeriod { + result := &TokenPeriod{} + return result.EnforceGracePeriod() +} + +func (it *TokenPeriod) AsSeconds() (int, int, bool) { + if it == nil { + return asSeconds(15), asSeconds(5), false + } + it.EnforceGracePeriod() + return asSeconds(it.ValidityTime), asSeconds(it.GracePeriod), true +} + +func (it *TokenPeriod) Liveline() int64 { + valid, _, _ := it.AsSeconds() + when := time.Now().Unix() + return when + int64(valid) +} + +func (it *TokenPeriod) Deadline() int64 { + valid, grace, _ := it.AsSeconds() + when := time.Now().Unix() + return when + int64(valid+grace) +} + +func (it *TokenPeriod) RequestSeconds() int { + valid, grace, _ := it.AsSeconds() + return int(valid + grace) +} + +func FreezeEnvironmentListing(label string, config robot.Robot) { + goldenfile := conda.GoldenMasterFilename(label) + listing := conda.LoadWantedDependencies(goldenfile) + if len(listing) == 0 { + common.Log("No dependencies found at %q", goldenfile) + return + } + env, err := conda.ReadPackageCondaYaml(config.CondaConfigFile()) if err != nil { - return false + common.Log("Could not read %q, reason: %v", config.CondaConfigFile(), err) + return + } + frozen := env.FreezeDependencies(listing) + err = frozen.SaveAs(config.FreezeFilename()) + if err != nil { + common.Log("Could not save %q, reason: %v", config.FreezeFilename(), err) + } +} + +func ExecutionEnvironmentListing(wantedfile, label string, searchPath pathlib.PathParts, directory, outputDir string, environment []string) bool { + common.Timeline("execution environment listing") + defer common.Log("--") + goldenfile := conda.GoldenMasterFilename(label) + err := conda.SideBySideViewOfDependencies(goldenfile, wantedfile) + if err != nil { + pip, ok := searchPath.Which("pip", conda.FileExtensions) + if !ok { + return false + } + fullPip, err := filepath.EvalSymlinks(pip) + if err != nil { + return false + } + common.Log("Installed pip packages:") + if common.NoOutputCapture { + _, err = shell.New(environment, directory, fullPip, "freeze", "--all").Execute(false) + } else { + _, err = shell.New(environment, directory, fullPip, "freeze", "--all").Tee(outputDir, false) + } } - common.Log("Installed pip packages:") - _, err = shell.New(environment, directory, fullPip, "freeze", "--all").Tee(outputDir, false) if err != nil { return false } - common.Log("--") return true } +func LoadAnyTaskEnvironment(packfile string, force bool) (bool, robot.Robot, robot.Task, string) { + FixRobot(packfile) + config, err := robot.LoadRobotYaml(packfile, false) + if err != nil { + pretty.Exit(1, "Error: %v", err) + } + anytasks := config.AvailableTasks() + if len(anytasks) == 0 { + pretty.Exit(1, "Could not find tasks from %q.", packfile) + } + return LoadTaskWithEnvironment(packfile, anytasks[0], force) +} + func LoadTaskWithEnvironment(packfile, theTask string, force bool) (bool, robot.Robot, robot.Task, string) { + common.Timeline("task environment load started") FixRobot(packfile) - config, err := robot.LoadYamlConfiguration(packfile) + config, err := robot.LoadRobotYaml(packfile, true) if err != nil { pretty.Exit(1, "Error: %v", err) } @@ -57,24 +169,49 @@ func LoadTaskWithEnvironment(packfile, theTask string, force bool) (bool, robot. todo := config.TaskByName(theTask) if todo == nil { - pretty.Exit(3, "Error: Could not resolve task to run. Available tasks are: %v", config.AvailableTasks()) + pretty.Exit(3, "Error: Could not resolve what task to run. Select one using --task option.\nAvailable task names are: %v.", strings.Join(config.AvailableTasks(), ", ")) + } + + if config.HasHolozip() && !common.UsesHolotree() { + pretty.Exit(4, "Error: this robot requires holotree, but no --space was given!") + } + + pathlib.EnsureDirectoryExists(config.ArtifactDirectory()) + journal.ForRun(filepath.Join(config.ArtifactDirectory(), "journal.run")) + cache, err := SummonCache() + if err == nil && len(cache.Userset()) > 1 { + pretty.Note("There seems to be multiple users sharing %s, which might cause problems.", common.Product.HomeVariable()) + pretty.Note("These are the users: %s.", cache.Userset()) + pretty.Highlight("To correct this problem, make sure that there is only one user per %s.", common.Product.HomeVariable()) + common.RunJournal("sharing", fmt.Sprintf("name=%s from=%s users=%s", theTask, packfile, cache.Userset()), fmt.Sprintf("multiple users shareing %s", common.Product.HomeVariable())) } + common.RunJournal("start task", fmt.Sprintf("name=%s from=%s", theTask, packfile), "at task environment setup") + if !config.UsesConda() { return true, config, todo, "" } - label, err := conda.NewEnvironment(force, config.CondaConfigFile()) + label, _, err := htfs.NewEnvironment(config.CondaConfigFile(), config.Holozip(), true, force, PullCatalog) if err != nil { + pretty.RccPointOfView(newEnvironment, err) pretty.Exit(4, "Error: %v", err) } return false, config, todo, label } func SelectExecutionModel(runFlags *RunFlags, simple bool, template []string, config robot.Robot, todo robot.Task, label string, interactive bool, extraEnv map[string]string) { + common.TimelineBegin("robot execution (simple=%v).", simple) + common.RunJournal("start", "robot", "started") + defer common.RunJournal("stop", "robot", "done") + defer common.TimelineEnd() + pathlib.EnsureDirectoryExists(config.ArtifactDirectory()) if simple { + common.RunJournal("select", "robot", "simple run") + pathlib.NoteDirectoryContent("[Before run] Artifact dir", config.ArtifactDirectory(), true) ExecuteSimpleTask(runFlags, template, config, todo, interactive, extraEnv) } else { + common.RunJournal("run", "robot", "task run") ExecuteTask(runFlags, template, config, todo, label, interactive, extraEnv) } } @@ -84,7 +221,7 @@ func ExecuteSimpleTask(flags *RunFlags, template []string, config robot.Robot, t task := make([]string, len(template)) copy(task, template) searchPath := pathlib.TargetPath() - searchPath = searchPath.Prepend(todo.Paths(config)...) + searchPath = searchPath.Prepend(config.Paths()...) found, ok := searchPath.Which(task[0], conda.FileExtensions) if !ok { pretty.Exit(6, "Error: Cannot find command: %v", task[0]) @@ -95,14 +232,14 @@ func ExecuteSimpleTask(flags *RunFlags, template []string, config robot.Robot, t } var data Token if len(flags.WorkspaceId) > 0 { - claims := RunClaims(flags.ValidityTime*60, flags.WorkspaceId) - data, err = AuthorizeClaims(flags.AccountName, claims) + claims := RunRobotClaims(flags.TokenPeriod.RequestSeconds(), flags.WorkspaceId) + data, err = AuthorizeClaims(flags.AccountName, claims, flags.TokenPeriod.EnforceGracePeriod()) } if err != nil { pretty.Exit(8, "Error: %v", err) } task[0] = fullpath - directory := todo.WorkingDirectory(config) + directory := config.WorkingDirectory() environment := robot.PlainEnvironment([]string{searchPath.AsEnvironmental("PATH")}, true) if len(data) > 0 { endpoint := data["endpoint"] @@ -120,15 +257,34 @@ func ExecuteSimpleTask(flags *RunFlags, template []string, config robot.Robot, t environment = append(environment, fmt.Sprintf("%s=%s", key, value)) } } - outputDir := todo.ArtifactDirectory(config) - common.Debug("DEBUG: about to run command - %v", task) - _, err = shell.New(environment, directory, task...).Tee(outputDir, interactive) + outputDir, err := pathlib.EnsureDirectory(config.ArtifactDirectory()) if err != nil { pretty.Exit(9, "Error: %v", err) } + common.Debug("about to run command - %v", task) + if common.NoOutputCapture { + _, err = shell.New(environment, directory, task...).Execute(interactive) + } else { + _, err = shell.New(environment, directory, task...).Tee(outputDir, interactive) + } + if err != nil { + pretty.Exit(10, "Error: %v", err) + } pretty.Ok() } +func findExecutableOrDie(searchPath pathlib.PathParts, executable string) string { + found, ok := searchPath.Which(executable, conda.FileExtensions) + if !ok { + pretty.Exit(6, "Error: Cannot find command: %v", executable) + } + fullpath, err := filepath.EvalSymlinks(found) + if err != nil { + pretty.Exit(7, "Error: %v", err) + } + return fullpath +} + func ExecuteTask(flags *RunFlags, template []string, config robot.Robot, todo robot.Task, label string, interactive bool, extraEnv map[string]string) { common.Debug("Command line is: %v", template) developmentEnvironment, err := robot.LoadEnvironmentSetup(flags.EnvironmentFile) @@ -137,26 +293,18 @@ func ExecuteTask(flags *RunFlags, template []string, config robot.Robot, todo ro } task := make([]string, len(template)) copy(task, template) - searchPath := todo.SearchPath(config, label) - found, ok := searchPath.Which(task[0], conda.FileExtensions) - if !ok { - pretty.Exit(6, "Error: Cannot find command: %v", task[0]) - } - fullpath, err := filepath.EvalSymlinks(found) - if err != nil { - pretty.Exit(7, "Error: %v", err) - } + searchPath := config.SearchPath(label) + task[0] = findExecutableOrDie(searchPath, task[0]) var data Token - if len(flags.WorkspaceId) > 0 { - claims := RunClaims(flags.ValidityTime*60, flags.WorkspaceId) - data, err = AuthorizeClaims(flags.AccountName, claims) + if !flags.Assistant && len(flags.WorkspaceId) > 0 { + claims := RunRobotClaims(flags.TokenPeriod.RequestSeconds(), flags.WorkspaceId) + data, err = AuthorizeClaims(flags.AccountName, claims, nil) } if err != nil { pretty.Exit(8, "Error: %v", err) } - task[0] = fullpath - directory := todo.WorkingDirectory(config) - environment := todo.ExecutionEnvironment(config, label, developmentEnvironment.AsEnvironment(), true) + directory := config.WorkingDirectory() + environment := config.RobotExecutionEnvironment(label, developmentEnvironment.AsEnvironment(), true) if len(data) > 0 { endpoint := data["endpoint"] for _, key := range rcHosts { @@ -173,14 +321,72 @@ func ExecuteTask(flags *RunFlags, template []string, config robot.Robot, todo ro environment = append(environment, fmt.Sprintf("%s=%s", key, value)) } } - outputDir := todo.ArtifactDirectory(config) - if !common.Silent && !interactive { - PipFreeze(searchPath, directory, outputDir, environment) - } - common.Debug("DEBUG: about to run command - %v", task) - _, err = shell.New(environment, directory, task...).Tee(outputDir, interactive) + before := make(map[string]string) + beforeHash, beforeErr := conda.DigestFor(label, before) + outputDir, err := pathlib.EnsureDirectory(config.ArtifactDirectory()) if err != nil { pretty.Exit(9, "Error: %v", err) } + if !flags.NoPipFreeze && !flags.Assistant && !common.Silent() && !interactive { + wantedfile, _ := config.DependenciesFile() + ExecutionEnvironmentListing(wantedfile, label, searchPath, directory, outputDir, environment) + } + + pathlib.NoteDirectoryContent("[Before run] Artifact dir", config.ArtifactDirectory(), true) + + FreezeEnvironmentListing(label, config) + preRunScripts := config.PreRunScripts() + if !common.DeveloperFlag && preRunScripts != nil && len(preRunScripts) > 0 { + common.Timeline("pre run scripts started") + common.Debug("=== pre run script phase ===") + for _, script := range preRunScripts { + if !robot.PlatformAcceptableFile(runtime.GOARCH, runtime.GOOS, script) { + continue + } + scriptCommand, err := shell.Split(script) + if err != nil { + pretty.RccPointOfView(preRun, err) + pretty.Exit(11, "%sScript '%s' parsing failure: %v%s", pretty.Red, script, err, pretty.Reset) + } + scriptCommand[0] = findExecutableOrDie(searchPath, scriptCommand[0]) + common.Debug("Running pre run script '%s' ...", script) + _, err = shell.New(environment, directory, scriptCommand...).Execute(interactive) + if err != nil { + pretty.RccPointOfView(preRun, err) + pretty.Exit(12, "%sScript '%s' failure: %v%s", pretty.Red, script, err, pretty.Reset) + } + } + journal.CurrentBuildEvent().PreRunComplete() + common.Timeline("pre run scripts completed") + } + + common.Debug("about to run command - %v", task) + journal.CurrentBuildEvent().RobotStarts() + pipe := WatchChildren(os.Getpid(), 550*time.Millisecond) + shell.WithInterrupt(func() { + exitcode := 0 + if common.NoOutputCapture { + exitcode, err = shell.New(environment, directory, task...).Execute(interactive) + } else { + exitcode, err = shell.New(environment, directory, task...).Tee(outputDir, interactive) + } + if exitcode != 0 { + details := fmt.Sprintf("%s_%d_%08x", common.Platform(), exitcode, uint32(exitcode)) + cloud.InternalBackgroundMetric(common.ControllerIdentity(), "rcc.cli.run.failure", details) + } + }) + pretty.RccPointOfView(actualRun, err) + seen, ok := <-pipe + suberr := SubprocessWarning(seen, ok) + if suberr != nil { + pretty.Warning("Problem with subprocess warnings, reason: %v", suberr) + } + journal.CurrentBuildEvent().RobotEnds() + after := make(map[string]string) + afterHash, afterErr := conda.DigestFor(label, after) + conda.DiagnoseDirty(label, label, beforeHash, afterHash, beforeErr, afterErr, before, after, true) + if err != nil { + pretty.Exit(10, "Error: %v (robot run exit)", err) + } pretty.Ok() } diff --git a/operations/running_test.go b/operations/running_test.go new file mode 100644 index 00000000..03a8e69b --- /dev/null +++ b/operations/running_test.go @@ -0,0 +1,18 @@ +package operations_test + +import ( + "testing" + + "github.com/robocorp/rcc/hamlet" + "github.com/robocorp/rcc/operations" +) + +func TestTokenPeriodWorksAsExpected(t *testing.T) { + must, wont := hamlet.Specifications(t) + + var period *operations.TokenPeriod + must.Nil(period) + wont.Panic(func() { + period.Deadline() + }) +} diff --git a/operations/security_test.go b/operations/security_test.go index 9cae8850..1b5be482 100644 --- a/operations/security_test.go +++ b/operations/security_test.go @@ -9,21 +9,24 @@ import ( "github.com/robocorp/rcc/operations" ) -func TestCanCreatePrivateKey(t *testing.T) { +func TestCanCreatePrivateRsaKey(t *testing.T) { must, wont := hamlet.Specifications(t) - key, err := operations.GenerateEphemeralKey() + ephemeral, err := operations.GenerateEphemeralKey() must.Nil(err) + wont.Nil(ephemeral) + key, ok := ephemeral.(*operations.EncryptionV1) + must.True(ok) wont.Nil(key) wont.Nil(key.Public()) publicKey, ok := key.Public().(*rsa.PublicKey) must.True(ok) - must.Equal(512, publicKey.Size()) - must.Equal(704, len(key.PublicDER())) - must.Equal(775, len(key.PublicPEM())) + wont.Nil(publicKey) + must.Equal(256, publicKey.Size()) + must.Equal(426, len(key.PublicPEM())) body, err := key.RequestObject(nil) must.Nil(err) - must.Equal(847, len(body)) + must.Equal(493, len(body)) textual := string(body) must.True(strings.Contains(textual, "encryption")) must.True(strings.Contains(textual, "scheme")) diff --git a/operations/testdata/payload.txt b/operations/testdata/payload.txt new file mode 100644 index 00000000..c2981a99 --- /dev/null +++ b/operations/testdata/payload.txt @@ -0,0 +1 @@ +payload diff --git a/operations/tlscheck.go b/operations/tlscheck.go new file mode 100644 index 00000000..bca275b7 --- /dev/null +++ b/operations/tlscheck.go @@ -0,0 +1,454 @@ +package operations + +import ( + "context" + "crypto/md5" + "crypto/sha1" + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "encoding/pem" + "fmt" + "hash" + "net" + "net/http" + "os" + "strings" + "time" + + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/fail" + "github.com/robocorp/rcc/pretty" + "github.com/robocorp/rcc/set" + "github.com/robocorp/rcc/settings" + "gopkg.in/yaml.v2" +) + +type ( + tlsConfigs []*tls.Config + + UrlConfig struct { + TrustedURLs []string `yaml:"trusted,omitempty"` + UntrustedURLs []string `yaml:"untrusted,omitempty"` + } +) + +var ( + tlsVersions = map[uint16]string{} + knownVersions = []uint16{ + tls.VersionTLS13, + tls.VersionTLS12, + tls.VersionTLS11, + tls.VersionTLS10, + tls.VersionSSL30, + } +) + +func init() { + tlsVersions[tls.VersionSSL30] = "SSLv3" + tlsVersions[tls.VersionTLS10] = "TLS 1.0" + tlsVersions[tls.VersionTLS11] = "TLS 1.1" + tlsVersions[tls.VersionTLS12] = "TLS 1.2" + tlsVersions[tls.VersionTLS13] = "TLS 1.3" +} + +func tlsCheckHeadOnly(url string) (*tls.ConnectionState, error) { + transport := settings.Global.ConfiguredHttpTransport() + transport.TLSClientConfig.InsecureSkipVerify = true + transport.TLSClientConfig.MinVersion = tls.VersionSSL30 + // above two setting are needed for TLS checks + // they weaken security, and that is why this code is only used + // to get TLS connection state and nothing else + // this is intentional, so that network diagnosis can detect + // unsecure certificates, and connections to weaker TLS version + // [ref: Github CodeQL security warning] + client := http.Client{ + Transport: transport, + Timeout: 3 * time.Second, + } + response, err := client.Head(url) + if err != nil { + return nil, err + } + if response.TLS == nil { + return nil, fmt.Errorf("Strange state, could not get TLS information from URL %q and there was no error.", url) + } + return response.TLS, nil +} + +func certificateChain(certificates []*x509.Certificate) string { + parts := make([]string, 0, len(certificates)) + for at, certificate := range certificates { + names := strings.Join(certificate.DNSNames, ", ") + before := certificate.NotBefore.Format("2006-Jan-02") + after := certificate.NotAfter.Format("2006-Jan-02") + form := fmt.Sprintf("#%d: [% 02X ...] names [%s] %s...%s %q issued by %q", at, certificate.Signature[:6], names, before, after, certificate.Subject, certificate.Issuer) + parts = append(parts, form) + } + return strings.Join(parts, "; ") +} + +func tlsCheckHost(host string, roots map[string]bool) []*common.DiagnosticCheck { + transport := settings.Global.ConfiguredHttpTransport() + result := []*common.DiagnosticCheck{} + supportNetworkUrl := settings.Global.DocsLink("troubleshooting/firewall-and-proxies") + url := fmt.Sprintf("https://%s/", host) + state, err := tlsCheckHeadOnly(url) + if err != nil { + result = append(result, &common.DiagnosticCheck{ + Type: "network", + Category: common.CategoryNetworkLink, + Status: statusWarning, + Message: fmt.Sprintf("%s -> %v", url, err), + Link: supportNetworkUrl, + }) + return result + } + server := state.ServerName + version, ok := tlsVersions[state.Version] + if !ok { + result = append(result, &common.DiagnosticCheck{ + Type: "network", + Category: common.CategoryNetworkTLSVersion, + Status: statusWarning, + Message: fmt.Sprintf("unknown TLS version: %q -> %03x", host, state.Version), + Link: supportNetworkUrl, + }) + } else { + tlsStatus := statusOk + if state.Version < tls.VersionTLS12 { + tlsStatus = statusWarning + } + result = append(result, &common.DiagnosticCheck{ + Type: "network", + Category: common.CategoryNetworkTLSVersion, + Status: tlsStatus, + Message: fmt.Sprintf("TLS version: %q -> %s", host, version), + Link: supportNetworkUrl, + }) + } + toVerify := x509.VerifyOptions{ + DNSName: server, + Roots: transport.TLSClientConfig.RootCAs, + Intermediates: x509.NewCertPool(), + } + certificates := state.PeerCertificates + if len(certificates) == 0 { + result = append(result, &common.DiagnosticCheck{ + Type: "network", + Category: common.CategoryNetworkTLSVerify, + Status: statusWarning, + Message: fmt.Sprintf("no certificates for %s", server), + Link: supportNetworkUrl, + }) + return result + } + last := certificates[0] + for _, certificate := range certificates[1:] { + toVerify.Intermediates.AddCert(certificate) + last = certificate + } + _, err = certificates[0].Verify(toVerify) + roots[last.Issuer.String()] = err == nil + if err != nil { + result = append(result, &common.DiagnosticCheck{ + Type: "network", + Category: common.CategoryNetworkTLSVerify, + Status: statusWarning, + Message: fmt.Sprintf("TLS verification of %q failed, reason: %v [last issuer: %q]", server, err, last.Issuer), + Link: supportNetworkUrl, + }) + if common.DebugFlag() { + result = append(result, &common.DiagnosticCheck{ + Type: "network", + Category: common.CategoryNetworkTLSChain, + Status: statusWarning, + Message: fmt.Sprintf("%q certificate chain is {%s}.", host, certificateChain(certificates)), + Link: supportNetworkUrl, + }) + } + } else { + result = append(result, &common.DiagnosticCheck{ + Type: "network", + Category: common.CategoryNetworkTLSVerify, + Status: statusOk, + Message: fmt.Sprintf("TLS verification of %q passed with certificate issued by %q", server, last.Issuer), + Link: supportNetworkUrl, + }) + } + return result +} + +func configurationVariations(root *x509.CertPool) tlsConfigs { + configs := make(tlsConfigs, len(knownVersions)) + for at, version := range knownVersions { + configs[at] = &tls.Config{ + InsecureSkipVerify: true, + RootCAs: root, + MinVersion: version, + MaxVersion: version, + } + } + return configs +} + +func formatFingerprint(digest hash.Hash, certificate *x509.Certificate) string { + if certificate == nil { + return "N/A" + } + digest.Write(certificate.Raw) + return strings.Replace(fmt.Sprintf("% 02x", digest.Sum(nil)), " ", ":", -1) +} + +func md5Fingerprint(certificate *x509.Certificate) string { + return formatFingerprint(md5.New(), certificate) +} + +func sha1Fingerprint(certificate *x509.Certificate) string { + return formatFingerprint(sha1.New(), certificate) +} + +func sha256Fingerprint(certificate *x509.Certificate) string { + return formatFingerprint(sha256.New(), certificate) +} + +func certificateFingerprint(certificate *x509.Certificate) string { + if certificate == nil { + return "[nil]" + } + digest := sha256.New() + digest.Write(certificate.Raw) + sum := digest.Sum(nil) + return fmt.Sprintf("[% 02X ...]", sum[:16]) +} + +func dnsLookup(serverport string) string { + parts := strings.Split(serverport, ":") + if len(parts) == 0 { + return "DNS: []" + } + dns, err := net.LookupHost(parts[0]) + if err != nil { + return fmt.Sprintf("DNS [%v]", err) + } + return fmt.Sprintf("DNS %q", dns) +} + +func probeVersion(serverport string, config *tls.Config, seen map[string]int) { + dialer := &tls.Dialer{ + Config: config, + } + timeout, _ := context.WithTimeout(context.Background(), 5*time.Second) + intermediate, err := dialer.DialContext(timeout, "tcp", serverport) + if err != nil { + common.Log(" %s%s failed, reason: %v%s", pretty.Yellow, tlsVersions[config.MinVersion], err, pretty.Reset) + return + } + defer intermediate.Close() + conn, ok := intermediate.(*tls.Conn) + if !ok { + common.Log(" %s%s failed, reason: could not covert to TLS connection.%s", pretty.Yellow, tlsVersions[config.MinVersion], pretty.Reset) + return + } + state := conn.ConnectionState() + cipher := tls.CipherSuiteName(state.CipherSuite) + version, ok := tlsVersions[state.Version] + if !ok { + version = fmt.Sprintf("unknown: %03x", state.Version) + } + server := state.ServerName + toVerify := x509.VerifyOptions{ + DNSName: server, + Roots: config.RootCAs, + Intermediates: x509.NewCertPool(), + } + common.Log(" %s%s supported, cipher suite %q, server: %q, address: %q%s", pretty.Green, version, cipher, server, conn.RemoteAddr(), pretty.Reset) + certificates := state.PeerCertificates + before := len(seen) + for at, certificate := range certificates { + if at > 0 { + toVerify.Intermediates.AddCert(certificate) + } + fingerprint := certificateFingerprint(certificate) + hit, ok := seen[fingerprint] + if ok { + common.Log(" %s#%d: [ID:%d] again %s%s", pretty.Grey, at, hit, fingerprint, pretty.Reset) + continue + } + hit = len(seen) + 1 + seen[fingerprint] = hit + names := strings.Join(certificate.DNSNames, ", ") + before := certificate.NotBefore.Format("2006-Jan-02") + after := certificate.NotAfter.Format("2006-Jan-02") + common.Log(" #%d: %s[ID:%d]%s %s %s - %s [%s]", at, pretty.Magenta, hit, pretty.Reset, fingerprint, before, after, names) + common.Log(" + subject %s", certificate.Subject) + common.Log(" + issuer %s", certificate.Issuer) + } + if len(seen) == before { + return + } + _, err = certificates[0].Verify(toVerify) + if err != nil { + common.Log(" %s!!! verification failure: %v%s", pretty.Red, err, pretty.Reset) + } +} + +func probeServer(index int, serverport string, variations tlsConfigs, seen map[string]int) { + common.Log("%s#%d: Server %q, %s%s", pretty.Cyan, index, serverport, dnsLookup(serverport), pretty.Reset) + for _, variation := range variations { + probeVersion(serverport, variation, seen) + } +} + +func TLSProbe(serverports []string) (err error) { + defer fail.Around(&err) + + root, err := x509.SystemCertPool() + fail.On(err != nil, "Cannot get system certificate pool, reason: %v", err) + + seen := make(map[string]int) + + variations := configurationVariations(root) + for at, serverport := range serverports { + if at > 0 { + common.Log("--") + } + probeServer(at+1, serverport, variations, seen) + } + return nil +} + +func urlConfigFrom(content []byte) (*UrlConfig, error) { + result := new(UrlConfig) + err := yaml.Unmarshal(content, result) + if err != nil { + return nil, err + } + return result, nil +} + +func mergeConfigFiles(configfiles []string) (trusted []string, untrusted []string, err error) { + defer fail.Around(&err) + trusted, untrusted = []string{}, []string{} + for _, filename := range configfiles { + content, err := os.ReadFile(filename) + fail.On(err != nil, "Failed to read %q, reason: %v", filename, err) + config, err := urlConfigFrom(content) + fail.On(err != nil, "Failed to parse %q, reason: %v", filename, err) + trusted = append(trusted, config.TrustedURLs...) + untrusted = append(untrusted, config.UntrustedURLs...) + } + return trusted, untrusted, nil +} + +func certificateExport(certificate *x509.Certificate, trusted bool) (text string, err error) { + defer fail.Around(&err) + + label := "!UNTRUSTED" + if trusted { + label = "trusted" + } + + stream := &strings.Builder{} + if certificate != nil { + fmt.Fprintf(stream, "# Category: %q with flags:%010b (rcc %s)\n", label, certificate.KeyUsage, common.Version) + fmt.Fprintln(stream, "# Issuer:", certificate.Issuer) + fmt.Fprintln(stream, "# Subject:", certificate.Subject) + fmt.Fprintf(stream, "# Label: %q\n", certificate.Subject.CommonName) + fmt.Fprintf(stream, "# Serial: %d\n", certificate.SerialNumber) + fmt.Fprintf(stream, "# MD5 Fingerprint: %s\n", md5Fingerprint(certificate)) + fmt.Fprintf(stream, "# SHA1 Fingerprint: %s\n", sha1Fingerprint(certificate)) + fmt.Fprintf(stream, "# SHA256 Fingerprint: %s\n", sha256Fingerprint(certificate)) + block := &pem.Block{Type: "CERTIFICATE", Bytes: certificate.Raw} + err := pem.Encode(stream, block) + fail.On(err != nil, "Could not PEM encode certificate, reason: %v", err) + } + return stream.String(), nil +} + +func pickVerifiedCertificates(roots *x509.CertPool, candidates []*x509.Certificate) ([]*x509.Certificate, []error) { + errors := make([]error, 0, len(candidates)) + verified := make([]*x509.Certificate, 0, len(candidates)) + toVerify := x509.VerifyOptions{Roots: roots} + for _, candidate := range candidates { + _, err := candidate.Verify(toVerify) + if err == nil { + verified = append(verified, candidate) + } else { + errors = append(errors, err) + } + } + return verified, errors +} + +func tlsExportUrls(roots *x509.CertPool, unique map[string]bool, urls []string, untrusted bool) (ok bool, err error) { + defer fail.Around(&err) + ok = true +search: + for _, url := range urls { + state, err := tlsCheckHeadOnly(url) + if err != nil { + ok = false + pretty.Warning("Failed to check URL %q for TLS certificates, reason: %v", url, err) + continue search + } + if state != nil { + total := len(state.PeerCertificates) + if total < 1 { + ok = false + pretty.Warning("Failed to check URL %q for TLS certificates, reason: too few certificates in chain", url) + continue search + } + exportable := state.PeerCertificates + if !untrusted { + verified, errors := pickVerifiedCertificates(roots, state.PeerCertificates) + if len(verified) == 0 { + pretty.Warning("Failed to verify any of TLS certificates for URL %q, reasons:", url) + for _, err := range errors { + pretty.Warning("- %v", err) + } + ok = false + continue search + } + exportable = verified + } + for _, export := range exportable { + text, err := certificateExport(export, !untrusted) + if err != nil { + pretty.Warning("Failed to export TLS certificates for URL %q, reason: %v", url, err) + ok = false + continue search + } + unique[text] = true + } + } else { + pretty.Warning("URL %q does not have TLS available!", url) + ok = false + } + } + return ok, nil +} + +func TLSExport(filename string, configfiles []string) (err error) { + defer fail.Around(&err) + + roots, err := x509.SystemCertPool() + fail.On(err != nil, "Cannot get system certificate pool, reason: %v", err) + + trustedURLs, untrustedURLs, err := mergeConfigFiles(configfiles) + fail.Fast(err) + + unique := make(map[string]bool) + trusted, err := tlsExportUrls(roots, unique, trustedURLs, false) + fail.Fast(err) + untrusted, err := tlsExportUrls(roots, unique, untrustedURLs, true) + fail.Fast(err) + if trusted && untrusted && len(unique) > 0 { + fullset := strings.Join(set.Keys(unique), "\n") + err := os.WriteFile(filename, []byte(fullset), 0o600) + fail.On(err != nil, "Failed to write certificate export file %q, reason: %v", filename, err) + pretty.Highlight("Exported total of %d certificates into %q.", len(unique), filename) + return nil + } + return fmt.Errorf("Failed to export certificates. Reason unknown, maybe visible above. Flags are trusted:%v, untrusted:%v, count:%d.", trusted, untrusted, len(unique)) +} diff --git a/operations/updownload.go b/operations/updownload.go index b2637b46..0898fc34 100644 --- a/operations/updownload.go +++ b/operations/updownload.go @@ -7,10 +7,10 @@ import ( "net/url" "os" "path/filepath" - "time" "github.com/robocorp/rcc/cloud" "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/pathlib" ) const ( @@ -21,8 +21,8 @@ func linkFor(direction, workspaceId, robotId string) string { return fmt.Sprintf(loadLink, workspaceId, robotId, direction) } -func fetchRobotToken(client cloud.Client, account *account, claims *Claims) (string, error) { - data, err := AuthorizeCommand(client, account, claims) +func fetchRobotToken(client cloud.Client, account *account, claims *Claims, period *TokenPeriod) (string, error) { + data, err := AuthorizeCommand(client, account, claims, period) if err != nil { return "", err } @@ -34,21 +34,23 @@ func fetchRobotToken(client cloud.Client, account *account, claims *Claims) (str } func summonAssistantToken(client cloud.Client, account *account, workspaceId string) (string, error) { - claims := AssistantClaims(30*60, workspaceId) - token, ok := account.Cached(claims.Name, claims.Url) + period := DefaultTokenPeriod() + claims := RunAssistantClaims(period.RequestSeconds(), workspaceId) + token, ok := account.Cached(period, claims.Name, claims.Url) if ok { return token, nil } - return fetchRobotToken(client, account, claims) + return fetchRobotToken(client, account, claims, period) } -func summonRobotToken(client cloud.Client, account *account, workspaceId string) (string, error) { - claims := RobotClaims(30*60, workspaceId) - token, ok := account.Cached(claims.Name, claims.Url) +func summonGetRobotToken(client cloud.Client, account *account, workspaceId string) (string, error) { + period := DefaultTokenPeriod() + claims := GetRobotClaims(period.RequestSeconds(), workspaceId) + token, ok := account.Cached(period, claims.Name, claims.Url) if ok { return token, nil } - return fetchRobotToken(client, account, claims) + return fetchRobotToken(client, account, claims, period) } func getAnyloadLink(client cloud.Client, cloudUrl, credentials string) (string, error) { @@ -56,7 +58,7 @@ func getAnyloadLink(client cloud.Client, cloudUrl, credentials string) (string, request.Headers[authorization] = BearerToken(credentials) response := client.Get(request) if response.Status != 200 { - return "", errors.New(fmt.Sprintf("%d: %s", response.Status, response.Body)) + return "", fmt.Errorf("%d: %s", response.Status, response.Body) } token := make(Token) err := json.Unmarshal(response.Body, &token) @@ -65,11 +67,11 @@ func getAnyloadLink(client cloud.Client, cloudUrl, credentials string) (string, } uri, ok := token["uri"] if !ok { - return "", errors.New(fmt.Sprintf("Cannot find URI from %s.", response.Body)) + return "", fmt.Errorf("Cannot find URI from %s.", response.Body) } converted, ok := uri.(string) if !ok { - return "", errors.New(fmt.Sprintf("Cannot find URI as string from %s.", response.Body)) + return "", fmt.Errorf("Cannot find URI as string from %s.", response.Body) } return converted, nil } @@ -90,13 +92,13 @@ func putContent(client cloud.Client, awsUrl, zipfile string) error { request.Body = handle response := client.Put(request) if response.Status != 200 { - return errors.New(fmt.Sprintf("%d: %s", response.Status, response.Body)) + return fmt.Errorf("%d: %s", response.Status, response.Body) } return nil } func getContent(client cloud.Client, awsUrl, zipfile string) error { - handle, err := os.Create(zipfile) + handle, err := pathlib.Create(zipfile) if err != nil { return err } @@ -105,13 +107,13 @@ func getContent(client cloud.Client, awsUrl, zipfile string) error { request.Stream = handle response := client.Get(request) if response.Status != 200 { - return errors.New(fmt.Sprintf("%d: %s", response.Status, response.Body)) + return fmt.Errorf("%d: %s", response.Status, response.Body) } return nil } func UploadCommand(client cloud.Client, account *account, workspaceId, robotId, zipfile string, debug bool) error { - token, err := summonRobotToken(client, account, workspaceId) + token, err := summonEditRobotToken(client, account, workspaceId) if err != nil { return err } @@ -136,7 +138,8 @@ func UploadCommand(client cloud.Client, account *account, workspaceId, robotId, } func DownloadCommand(client cloud.Client, account *account, workspaceId, robotId, zipfile string, debug bool) error { - token, err := summonRobotToken(client, account, workspaceId) + common.Timeline("download started: %s", zipfile) + token, err := summonGetRobotToken(client, account, workspaceId) if err != nil { return err } @@ -161,12 +164,13 @@ func DownloadCommand(client cloud.Client, account *account, workspaceId, robotId } func SummonRobotZipfile(client cloud.Client, account *account, workspaceId, robotId, digest string) (string, error) { + common.Timeline("summon networked/cached robot.zip") found, ok := LookupRobot(digest) if ok { return found, nil } - zipfile := filepath.Join(os.TempDir(), fmt.Sprintf("summon%x.zip", time.Now().Unix())) - err := DownloadCommand(client, account, workspaceId, robotId, zipfile, common.DebugFlag) + zipfile := filepath.Join(pathlib.TempDir(), fmt.Sprintf("summon%x.zip", common.When)) + err := DownloadCommand(client, account, workspaceId, robotId, zipfile, common.DebugFlag()) if err != nil { return "", err } diff --git a/operations/workspaces.go b/operations/workspaces.go index 8fd97978..2fa59116 100644 --- a/operations/workspaces.go +++ b/operations/workspaces.go @@ -26,8 +26,8 @@ type RobotData struct { Package map[string]interface{} `json:"package,omitempty"` } -func fetchAnyToken(client cloud.Client, account *account, claims *Claims) (string, error) { - data, err := AuthorizeCommand(client, account, claims) +func fetchAnyToken(client cloud.Client, account *account, claims *Claims, period *TokenPeriod) (string, error) { + data, err := AuthorizeCommand(client, account, claims, period) if err != nil { return "", err } @@ -38,22 +38,24 @@ func fetchAnyToken(client cloud.Client, account *account, claims *Claims) (strin return "", errors.New("Could not get authorization token.") } -func summonActivityToken(client cloud.Client, account *account, workspace string) (string, error) { - claims := ActivityClaims(15*60, workspace) - token, ok := account.Cached(claims.Name, claims.Url) +func summonEditRobotToken(client cloud.Client, account *account, workspace string) (string, error) { + period := DefaultTokenPeriod() + claims := EditRobotClaims(period.RequestSeconds(), workspace) + token, ok := account.Cached(period, claims.Name, claims.Url) if ok { return token, nil } - return fetchAnyToken(client, account, claims) + return fetchAnyToken(client, account, claims, period) } func summonWorkspaceToken(client cloud.Client, account *account) (string, error) { - claims := WorkspaceTreeClaims(15 * 60) - token, ok := account.Cached(claims.Name, claims.Url) + period := DefaultTokenPeriod() + claims := ViewWorkspacesClaims(period.RequestSeconds()) + token, ok := account.Cached(period, claims.Name, claims.Url) if ok { return token, nil } - return fetchAnyToken(client, account, claims) + return fetchAnyToken(client, account, claims, period) } func WorkspacesCommand(client cloud.Client, account *account) (interface{}, error) { @@ -65,7 +67,7 @@ func WorkspacesCommand(client cloud.Client, account *account) (interface{}, erro request.Headers[authorization] = BearerToken(credentials) response := client.Get(request) if response.Status != 200 { - return nil, errors.New(fmt.Sprintf("%d: %s", response.Status, response.Body)) + return nil, fmt.Errorf("%d: %s", response.Status, response.Body) } tokens := make([]Token, 100) err = json.Unmarshal(response.Body, &tokens) @@ -84,7 +86,7 @@ func WorkspaceTreeCommandRequest(client cloud.Client, account *account, workspac request.Headers[authorization] = BearerToken(credentials) response := client.Get(request) if response.Status != 200 { - return nil, errors.New(fmt.Sprintf("%d: %s", response.Status, response.Body)) + return nil, fmt.Errorf("%d: %s", response.Status, response.Body) } return response, nil } @@ -102,37 +104,8 @@ func WorkspaceTreeCommand(client cloud.Client, account *account, workspace strin return token, nil } -func RobotDigestCommand(client cloud.Client, account *account, workspaceId, robotId string) (string, error) { - treedata, err := AssistantTreeCommand(client, account, workspaceId) - if err != nil { - return "", err - } - if treedata.Robots == nil { - return "", errors.New(fmt.Sprintf("Cannot find any valid robots from workspace %v!", workspaceId)) - } - var selected *RobotData = nil - for _, robot := range treedata.Robots { - if robot.Id == robotId { - selected = robot - break - } - } - if selected == nil { - return "", errors.New(fmt.Sprintf("Cannot find robot %v from workspace %v!", robotId, workspaceId)) - } - found, ok := selected.Package["sha256"] - if !ok { - return "", errors.New(fmt.Sprintf("Robot %v from workspace %v has no digest available!", robotId, workspaceId)) - } - digest, ok := found.(string) - if !ok { - return "", errors.New(fmt.Sprintf("Robot %v from workspace %v has no string digest available, only %#v!", robotId, workspaceId, found)) - } - return digest, nil -} - func NewRobotCommand(client cloud.Client, account *account, workspace, robotName string) (Token, error) { - credentials, err := summonActivityToken(client, account, workspace) + credentials, err := summonEditRobotToken(client, account, workspace) if err != nil { return nil, err } @@ -144,10 +117,11 @@ func NewRobotCommand(client cloud.Client, account *account, workspace, robotName } request := client.NewRequest(fmt.Sprintf(newRobotApi, workspace)) request.Headers[authorization] = BearerToken(credentials) + request.Headers[contentType] = applicationJson request.Body = strings.NewReader(body) response := client.Post(request) if response.Status != 200 { - return nil, errors.New(fmt.Sprintf("%d: %s", response.Status, response.Body)) + return nil, fmt.Errorf("%d: %s", response.Status, response.Body) } reply := make(Token) err = json.Unmarshal(response.Body, &reply) diff --git a/operations/zipper.go b/operations/zipper.go index 09c3146c..4c9d6c81 100644 --- a/operations/zipper.go +++ b/operations/zipper.go @@ -2,68 +2,124 @@ package operations import ( "archive/zip" - "errors" + "bytes" "fmt" "io" "os" "path/filepath" + "regexp" + "strings" "github.com/robocorp/rcc/common" "github.com/robocorp/rcc/pathlib" + "github.com/robocorp/rcc/pretty" "github.com/robocorp/rcc/robot" + "github.com/robocorp/rcc/set" ) -type WriteTarget struct { - Source *zip.File - Target string -} +const ( + backslash = `\` + slash = `/` +) + +var ( + libraryPattern = regexp.MustCompile("(?i)^library[/\\\\]{1,2}[0-9a-f]{2}[/\\\\]{1,2}[0-9a-f]{2}[/\\\\]{1,2}[0-9a-f]{2}[/\\\\]{1,2}[0-9a-f]{64}$") + catalogPattern = regexp.MustCompile("(?i)^catalog[/\\\\]{1,2}[0-9a-f]{16}v[0-9a-f]{2}\\.(?:windows|darwin|linux)_(?:amd64|arm64)") +) + +type ( + Verifier func(file *zip.File) error + + WriteTarget struct { + Source *zip.File + Target string + } + + Command interface { + Execute() bool + } + + CommandChannel chan Command + CompletedChannel chan bool +) -type Command interface { - Execute() bool +func slashed(text string) string { + return strings.Replace(text, backslash, slash, -1) } -type CommandChannel chan Command -type CompletedChannel chan bool +func HololibZipShape(file *zip.File) error { + library := libraryPattern.MatchString(file.Name) + catalog := catalogPattern.MatchString(file.Name) + if !library && !catalog { + return fmt.Errorf("filename %q does not match Holotree catalog or library entry pattern.", file.Name) + } + return nil +} func (it *WriteTarget) Execute() bool { - source, err := it.Source.Open() + err := it.execute() if err != nil { - return false + common.Error("zip extract", err) + common.Debug(" - failure with %q, reason: %v", it.Target, err) } - defer source.Close() - err = os.MkdirAll(filepath.Dir(it.Target), 0o750) + return err == nil +} + +func (it *WriteTarget) execute() error { + source, err := it.Source.Open() if err != nil { - return false + return err } - target, err := os.Create(it.Target) + defer source.Close() + target, err := pathlib.Create(it.Target) if err != nil { - return false + return err } defer target.Close() - common.Debug("- %v", it.Target) + common.Trace("- %v", it.Target) _, err = io.Copy(target, source) if err != nil { - common.Debug(" - failure: %v", err) + return err } os.Chtimes(it.Target, it.Source.Modified, it.Source.Modified) - return err == nil + return nil } type unzipper struct { - reader *zip.ReadCloser + reader *zip.Reader + closer io.Closer + flatten bool } func (it *unzipper) Close() { - it.reader.Close() + it.closer.Close() } -func newUnzipper(filename string) (*unzipper, error) { +func newPayloadUnzipper(filename string) (*unzipper, error) { + payloader, err := PayloadReaderAt(filename) + if err != nil { + return nil, err + } + reader, err := zip.NewReader(payloader, payloader.Limit()) + if err != nil { + return nil, err + } + return &unzipper{ + reader: reader, + closer: payloader, + flatten: false, + }, nil +} + +func newUnzipper(filename string, flatten bool) (*unzipper, error) { reader, err := zip.OpenReader(filename) if err != nil { return nil, err } return &unzipper{ - reader: reader, + reader: &reader.Reader, + closer: reader, + flatten: flatten, }, nil } @@ -79,6 +135,17 @@ func loopExecutor(work CommandChannel, done CompletedChannel) { done <- true } +func (it *unzipper) VerifyShape(verifier Verifier) []error { + errors := []error{} + for _, entry := range it.reader.File { + err := verifier(entry) + if err != nil { + errors = append(errors, err) + } + } + return errors +} + func (it *unzipper) Explode(workers int, directory string) error { // This is PoC code, for parallel extraction common.Debug("Exploding:") @@ -96,7 +163,7 @@ func (it *unzipper) Explode(workers int, directory string) error { } todo <- &WriteTarget{ Source: entry, - Target: filepath.Join(directory, entry.Name), + Target: filepath.Join(directory, slashed(entry.Name)), } } @@ -111,24 +178,85 @@ func (it *unzipper) Explode(workers int, directory string) error { return nil } +func (it *unzipper) Asset(name string) ([]byte, error) { + stream, err := it.reader.Open(name) + if err != nil { + return nil, err + } + defer stream.Close() + stat, err := stream.Stat() + if err != nil { + return nil, err + } + payload := bytes.NewBuffer(nil) + total, err := io.Copy(payload, stream) + if err != nil && err != io.EOF { + return nil, err + } + if int64(total) != stat.Size() { + pretty.Warning("Asset %q read partially!", name) + } + return payload.Bytes(), nil +} + +func (it *unzipper) ExtraDirectoryPrefixLength() (int, string) { + prefixes := make([]string, 0, 1) + for _, entry := range it.reader.File { + if entry.FileInfo().IsDir() { + continue + } + basename := filepath.Base(entry.Name) + if strings.ToLower(basename) != "robot.yaml" { + continue + } + dirname := filepath.Dir(entry.Name) + if len(dirname) > 0 && dirname != "." { + prefixes = append(prefixes, dirname) + } + } + prefixes = set.Set(prefixes) + if len(prefixes) != 1 { + return 0, "" + } + prefix := prefixes[0] + if len(prefix) == 0 { + return 0, "" + } + for _, entry := range it.reader.File { + if entry.FileInfo().IsDir() { + continue + } + if !strings.HasPrefix(entry.Name, prefix) { + return 0, "" + } + } + return len(prefix), prefix +} + func (it *unzipper) Extract(directory string) error { - common.Debug("Extracting:") - success := true + common.Trace("Extracting:") + limit, prefix := 0, "" + if it.flatten { + limit, prefix = it.ExtraDirectoryPrefixLength() + } + if limit > 0 { + pretty.Note("Flattening path %q out from extracted files.", prefix) + } for _, entry := range it.reader.File { if entry.FileInfo().IsDir() { continue } - target := filepath.Join(directory, entry.Name) + target := filepath.Join(directory, slashed(entry.Name)[limit:]) todo := WriteTarget{ Source: entry, Target: target, } - success = todo.Execute() && success - } - common.Debug("Done.") - if !success { - return errors.New(fmt.Sprintf("Problems while unwrapping robot. Use --debug to see details.")) + err := todo.execute() + if err != nil { + return fmt.Errorf("Problem while extracting zip, reason: %v", err) + } } + common.Trace("Done.") return nil } @@ -139,7 +267,7 @@ type zipper struct { } func newZipper(filename string) (*zipper, error) { - handle, err := os.Create(filename) + handle, err := pathlib.Create(filename) if err != nil { return nil, err } @@ -156,20 +284,39 @@ func (it *zipper) Note(err error) { common.Debug("Warning! %v", err) } -func (it *zipper) Add(fullpath, relativepath string, details os.FileInfo) { - common.Debug("- %v size %v", relativepath, details.Size()) +func ZipAppend(writer *zip.Writer, fullpath, relativepath string) error { source, err := os.Open(fullpath) if err != nil { - it.Note(err) - return + return err } defer source.Close() - target, err := it.writer.Create(relativepath) + target, err := writer.Create(slashed(relativepath)) + if err != nil { + return err + } + _, err = io.Copy(target, source) + return err +} + +func (it *zipper) Add(fullpath, relativepath string, details os.FileInfo) { + if details != nil { + common.Debug("- %v size %v", relativepath, details.Size()) + } else { + common.Debug("- %v", relativepath) + } + err := ZipAppend(it.writer, fullpath, relativepath) + if err != nil { + it.Note(err) + } +} + +func (it *zipper) AddBlob(relativepath string, blob []byte) { + target, err := it.writer.Create(slashed(relativepath)) if err != nil { it.Note(err) return } - _, err = io.Copy(target, source) + _, err = target.Write(blob) if err != nil { it.Note(err) } @@ -196,10 +343,11 @@ func defaultIgnores(selfie string) pathlib.Ignore { result = append(result, pathlib.IgnorePattern("temp/")) result = append(result, pathlib.IgnorePattern("tmp/")) result = append(result, pathlib.IgnorePattern("__pycache__")) + result = append(result, pathlib.IgnorePattern("__MACOSX")) return pathlib.CompositeIgnore(result...) } -func Unzip(directory, zipfile string, force, temporary bool) error { +func CarrierUnzip(directory, carrier string, force, temporary bool) error { fullpath, err := filepath.Abs(directory) if err != nil { return err @@ -212,7 +360,7 @@ func Unzip(directory, zipfile string, force, temporary bool) error { if err != nil { return err } - unzip, err := newUnzipper(zipfile) + unzip, err := newPayloadUnzipper(carrier) if err != nil { return err } @@ -224,16 +372,58 @@ func Unzip(directory, zipfile string, force, temporary bool) error { if temporary { return nil } - err = UpdateRobot(fullpath) + return FixDirectory(fullpath) +} + +func VerifyZip(zipfile string, verifier Verifier) []error { + common.TimelineBegin("zip verify %q [size: %s]", zipfile, pathlib.HumaneSize(zipfile)) + defer common.TimelineEnd() + + unzip, err := newUnzipper(zipfile, false) + if err != nil { + return []error{err} + } + defer unzip.Close() + + return unzip.VerifyShape(verifier) +} + +func Unzip(directory, zipfile string, force, temporary, flatten bool) error { + common.TimelineBegin("unzip %q [size: %s] to %q", zipfile, pathlib.HumaneSize(zipfile), directory) + defer common.TimelineEnd() + + fullpath, err := filepath.Abs(directory) if err != nil { return err } + if force { + err = pathlib.EnsureDirectoryExists(fullpath) + } else { + err = pathlib.EnsureEmptyDirectory(fullpath) + } + if err != nil { + return err + } + unzip, err := newUnzipper(zipfile, flatten) + if err != nil { + return err + } + defer unzip.Close() + err = unzip.Extract(fullpath) + if err != nil { + return err + } + if temporary { + return nil + } return FixDirectory(fullpath) } func Zip(directory, zipfile string, ignores []string) error { + common.Timeline("zip %q to %q", directory, zipfile) + defer common.Timeline("zip done") common.Debug("Wrapping %v into %v ...", directory, zipfile) - config, err := robot.LoadYamlConfiguration(robot.DetectConfigurationName(directory)) + config, err := robot.LoadRobotYaml(robot.DetectConfigurationName(directory), false) if err != nil { return err } @@ -248,6 +438,6 @@ func Zip(directory, zipfile string, ignores []string) error { return err } defaults := defaultIgnores(zipfile) - pathlib.Walk(directory, pathlib.CompositeIgnore(defaults, ignored), zipper.Add) + pathlib.ForceWalk(directory, pathlib.ForceFilename("hololib.zip"), pathlib.CompositeIgnore(defaults, ignored), zipper.Add) return nil } diff --git a/operations/zipper_test.go b/operations/zipper_test.go new file mode 100644 index 00000000..0a28c8ab --- /dev/null +++ b/operations/zipper_test.go @@ -0,0 +1,20 @@ +package operations + +import ( + "testing" + + "github.com/robocorp/rcc/hamlet" +) + +const ( + wintestpath = `a\b` + nixtestpath = `a/b` +) + +func TestCanConvertSlashes(t *testing.T) { + must, wont := hamlet.Specifications(t) + + wont.Equal(wintestpath, nixtestpath) + must.Equal(3, len(wintestpath)) + must.Equal(slashed(wintestpath), nixtestpath) +} diff --git a/pathlib/copyfile.go b/pathlib/copyfile.go index e27134e5..c12c2e44 100644 --- a/pathlib/copyfile.go +++ b/pathlib/copyfile.go @@ -8,9 +8,22 @@ import ( "github.com/robocorp/rcc/common" ) +type copyfunc func(io.Writer, io.Reader) (int64, error) + +type Copier func(string, string, bool) error + func CopyFile(source, target string, overwrite bool) error { - targetDir := filepath.Dir(target) - err := os.MkdirAll(targetDir, 0o755) + mark, err := Modtime(source) + if err != nil { + return err + } + err = copyFile(source, target, overwrite, io.Copy) + TouchWhen(target, mark) + return err +} + +func copyFile(source, target string, overwrite bool, copier copyfunc) error { + _, err := shared.MakeSharedDir(filepath.Dir(target)) if err != nil { return err } @@ -35,7 +48,7 @@ func CopyFile(source, target string, overwrite bool) error { } defer writable.Close() - _, err = io.Copy(writable, readable) + _, err = copier(writable, readable) if err != nil { common.Error("copy-file", err) } diff --git a/pathlib/finder.go b/pathlib/finder.go index 750561fc..c595e80f 100644 --- a/pathlib/finder.go +++ b/pathlib/finder.go @@ -1,7 +1,6 @@ package pathlib import ( - "errors" "fmt" "os" "path/filepath" @@ -27,8 +26,7 @@ func FindNamedPath(basedir, name string) (string, error) { return result[0], nil } if len(result) > 1 { - message := fmt.Sprintf("Found %d files named as '%s'. Expecting exactly one. %s", len(result), name, result) - return emptyString, errors.New(message) + return emptyString, fmt.Errorf("Found %d files named as '%s'. Expecting exactly one. %s", len(result), name, result) } if len(pending) > 0 { pending = append(pending, emptyString) @@ -54,6 +52,5 @@ func FindNamedPath(basedir, name string) (string, error) { } } } - message := fmt.Sprintf("Could not find path named '%s'.", name) - return emptyString, errors.New(message) + return emptyString, fmt.Errorf("Could not find path named '%s'.", name) } diff --git a/pathlib/functions.go b/pathlib/functions.go index 2a1469a3..651d961c 100644 --- a/pathlib/functions.go +++ b/pathlib/functions.go @@ -1,28 +1,128 @@ package pathlib import ( - "errors" "fmt" + "io/fs" + "math/rand" "os" "path/filepath" "time" + + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/fail" + "github.com/robocorp/rcc/pretty" ) +func TempDir() string { + base := os.TempDir() + _, err := EnsureDirectory(base) + if err != nil { + pretty.Warning("TempDir %q challenge, reason: %v", base, err) + } + return base +} + +func RestrictOwnerOnly(filename string) error { + return os.Chmod(filename, 0o600) +} + +func Create(filename string) (*os.File, error) { + _, err := EnsureParentDirectory(filename) + if err != nil { + return nil, fmt.Errorf("Failed to ensure that parent directories for %q exist, reason: %v", filename, err) + } + return os.Create(filename) +} + +func WriteFile(filename string, data []byte, mode os.FileMode) error { + if common.WarrantyVoided() { + return nil + } + _, err := EnsureParentDirectory(filename) + if err != nil { + return fmt.Errorf("Failed to ensure that parent directories for %q exist, reason: %v", filename, err) + } + return os.WriteFile(filename, data, mode) +} + +func Glob(directory string, pattern string) []string { + fullpath := filepath.Join(directory, pattern) + result, _ := filepath.Glob(fullpath) + return result +} + func Exists(pathname string) bool { _, err := os.Stat(pathname) return !os.IsNotExist(err) } +func Age(pathname string) uint64 { + var milliseconds int64 + stat, err := os.Stat(pathname) + if !os.IsNotExist(err) { + milliseconds = time.Now().Sub(stat.ModTime()).Milliseconds() + } + seconds := milliseconds / 1000 + if seconds < 0 { + return 0 + } + return uint64(seconds) +} + +func Abs(path string) (string, error) { + if filepath.IsAbs(path) { + return path, nil + } + fullpath, err := filepath.Abs(path) + if err != nil { + return "", err + } + return filepath.Clean(fullpath), nil +} + +func Symlink(pathname string) (string, bool) { + stat, err := os.Lstat(pathname) + if err != nil { + return "", false + } + mode := stat.Mode() + if mode&fs.ModeSymlink == 0 { + return "", false + } + name, err := os.Readlink(pathname) + if err != nil { + return "", false + } + return name, true +} + func IsDir(pathname string) bool { stat, err := os.Stat(pathname) + return err == nil && stat.IsDir() +} + +func IsEmptyDir(pathname string) bool { + if !IsDir(pathname) { + return false + } + content, err := os.ReadDir(pathname) if err != nil { return false } - return stat.IsDir() + return len(content) == 0 } func IsFile(pathname string) bool { - return Exists(pathname) && !IsDir(pathname) + stat, err := os.Stat(pathname) + return err == nil && !stat.IsDir() +} + +func DaysSinceModified(filename string) (int, error) { + stat, err := os.Stat(filename) + if err != nil { + return -1, err + } + return common.DayCountSince(stat.ModTime()), nil } func Size(pathname string) (int64, bool) { @@ -33,6 +133,35 @@ func Size(pathname string) (int64, bool) { return stat.Size(), true } +func kiloShift(size float64) float64 { + return size / 1024.0 +} + +func HumaneSizer(rawsize int64) (float64, string) { + kilos := kiloShift(float64(rawsize)) + if kilos < 1.0 { + return float64(rawsize), "b" + } + megas := kiloShift(kilos) + if megas < 1.0 { + return kilos, "K" + } + gigas := kiloShift(megas) + if gigas < 1.0 { + return megas, "M" + } + return gigas, "G" +} + +func HumaneSize(pathname string) string { + rawsize, ok := Size(pathname) + if !ok { + return "N/A" + } + value, suffix := HumaneSizer(rawsize) + return fmt.Sprintf("%3.1f%s", value, suffix) +} + func Modtime(pathname string) (time.Time, error) { stat, err := os.Stat(pathname) if err != nil { @@ -41,22 +170,167 @@ func Modtime(pathname string) (time.Time, error) { return stat.ModTime(), nil } -func EnsureDirectory(directory string) (string, error) { +func TryRemove(context, target string) (err error) { + for delay := 0; delay < 5; delay += 1 { + time.Sleep(time.Duration(delay*100) * time.Millisecond) + err = os.Remove(target) + if err == nil { + return nil + } + } + return fmt.Errorf("Remove failure [%s, %s, %s], reason: %s", context, common.ControllerIdentity(), common.HolotreeSpace, err) +} + +func TryRemoveAll(context, target string) (err error) { + for delay := 0; delay < 5; delay += 1 { + time.Sleep(time.Duration(delay*100) * time.Millisecond) + err = os.RemoveAll(target) + if err == nil { + return nil + } + } + return fmt.Errorf("RemoveAll failure [%s, %s, %s], reason: %s", context, common.ControllerIdentity(), common.HolotreeSpace, err) +} + +func TryRename(context, source, target string) (err error) { + for delay := 0; delay < 5; delay += 1 { + time.Sleep(time.Duration(delay*100) * time.Millisecond) + err = os.Rename(source, target) + if err == nil { + return nil + } + } + common.Debug("Heads up: rename about to fail [%q -> %q], reason: %s", source, target, err) + origin := "source" + intermediate := fmt.Sprintf("%s.%d_%x", source, os.Getpid(), rand.Intn(4096)) + err = os.Rename(source, intermediate) + if err == nil { + source = intermediate + origin = "target" + } + for delay := 0; delay < 5; delay += 1 { + time.Sleep(time.Duration(delay*100) * time.Millisecond) + err = os.Rename(source, target) + if err == nil { + return nil + } + } + return fmt.Errorf("Rename failure [%s, %s, %s, %s], reason: %s", context, common.ControllerIdentity(), common.HolotreeSpace, origin, err) +} + +func hasCorrectMode(stat fs.FileInfo, expected fs.FileMode) bool { + return expected == (stat.Mode() & expected) +} + +func ensureCorrectMode(fullpath string, stat fs.FileInfo, correct fs.FileMode) (string, error) { + if hasCorrectMode(stat, correct) { + return fullpath, nil + } + err := os.Chmod(fullpath, correct) + if err != nil { + return "", err + } + return fullpath, nil +} + +func makeModedDir(fullpath string, correct fs.FileMode) (path string, err error) { + defer fail.Around(&err) + + if common.WarrantyVoided() { + return fullpath, nil + } + + stat, err := os.Stat(fullpath) + if err == nil && stat.IsDir() { + return ensureCorrectMode(fullpath, stat, correct) + } + fail.On(err == nil, "Path %q exists, but is not a directory!", fullpath) + _, err = shared.MakeSharedDir(filepath.Dir(fullpath)) + fail.On(err != nil, "%v", err) + err = os.Mkdir(fullpath, correct) + fail.On(err != nil, "Failed to create directory %q, reason: %v", fullpath, err) + stat, err = os.Stat(fullpath) + fail.On(err != nil, "Failed to stat created directory %q, reason: %v", fullpath, err) + _, err = ensureCorrectMode(fullpath, stat, correct) + fail.On(err != nil, "Failed to make created directory shared %q, reason: %v", fullpath, err) + return fullpath, nil +} + +func MakeSharedFile(fullpath string) (string, error) { + return shared.MakeSharedFile(fullpath) +} + +func MakeSharedDir(fullpath string) (string, error) { + return shared.MakeSharedDir(fullpath) +} + +func ForceSharedDir(fullpath string) (string, error) { + return makeModedDir(fullpath, 0777) +} + +func IsSharedDir(fullpath string) bool { + stat, err := os.Stat(fullpath) + if err != nil { + return false + } + return stat.IsDir() && hasCorrectMode(stat, 0777) +} + +func doEnsureDirectory(directory string, mode fs.FileMode) (string, error) { fullpath, err := filepath.Abs(directory) if err != nil { return "", err } - err = os.MkdirAll(fullpath, 0o750) + if common.WarrantyVoided() || IsDir(fullpath) { + return fullpath, nil + } + err = os.MkdirAll(fullpath, mode) if err != nil { return "", err } stats, err := os.Stat(fullpath) if !stats.IsDir() { - return "", errors.New(fmt.Sprintf("Path %s is not a directory!", fullpath)) + return "", fmt.Errorf("Path %s is not a directory!", fullpath) } return fullpath, nil } +func EnsureSharedDirectory(directory string) (string, error) { + return shared.MakeSharedDir(directory) +} + +func EnsureSharedParentDirectory(resource string) (string, error) { + return EnsureSharedDirectory(filepath.Dir(resource)) +} + +func EnsureDirectory(directory string) (string, error) { + return doEnsureDirectory(directory, 0o750) +} + func EnsureParentDirectory(resource string) (string, error) { - return EnsureDirectory(filepath.Dir(resource)) + return doEnsureDirectory(filepath.Dir(resource), 0o750) +} + +func RemoveEmptyDirectores(starting string) (err error) { + defer fail.Around(&err) + + return DirWalk(starting, func(fullpath, relative string, entry os.FileInfo) { + if IsEmptyDir(fullpath) { + err = os.Remove(fullpath) + fail.On(err != nil, "%s", err) + } + }) +} + +func AppendFile(filename string, blob []byte) (err error) { + defer fail.Around(&err) + if common.WarrantyVoided() { + return nil + } + handle, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o644) + fail.On(err != nil, "Failed to open file %v -> %v", filename, err) + defer handle.Close() + _, err = handle.Write(blob) + fail.On(err != nil, "Failed to write file %v -> %v", filename, err) + return handle.Sync() } diff --git a/pathlib/lock.go b/pathlib/lock.go index 807e8518..52ddcc20 100644 --- a/pathlib/lock.go +++ b/pathlib/lock.go @@ -2,8 +2,10 @@ package pathlib import ( "os" + "time" "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/pretty" ) type Releaser interface { @@ -12,6 +14,7 @@ type Releaser interface { type Locked struct { *os.File + Latch chan bool } type fake bool @@ -24,3 +27,37 @@ func Fake() Releaser { common.Trace("LOCKER: lockless mode.") return fake(true) } + +func waitingLockNotification(lockfile, message string, latch chan bool) { + delay := 5 * time.Second + counter := 0 +waiting: + for { + select { + case <-latch: + return + case <-time.After(delay): + counter += 1 + delay *= 3 + pretty.Warning("#%d: %s (rcc lock wait)", counter, message) + common.Timeline("waiting for lock") + candidates, err := LockHoldersBy(lockfile) + if err != nil { + continue waiting + } + for _, candidate := range candidates { + message := candidate.Message() + pretty.Note(" : %s", message) + common.Timeline("+ %s", message) + } + } + } +} + +func LockWaitMessage(lockfile, message string) func() { + latch := make(chan bool) + go waitingLockNotification(lockfile, message, latch) + return func() { + latch <- true + } +} diff --git a/pathlib/lock_unix.go b/pathlib/lock_unix.go index c72d58bc..90272be0 100644 --- a/pathlib/lock_unix.go +++ b/pathlib/lock_unix.go @@ -1,3 +1,4 @@ +//go:build darwin || linux || !windows // +build darwin linux !windows package pathlib @@ -9,19 +10,30 @@ import ( "github.com/robocorp/rcc/common" ) -func Locker(filename string, trycount int) (Releaser, error) { - if Lockless { +func Locker(filename string, trycount int, sharedLocation bool) (Releaser, error) { + if common.WarrantyVoided() || Lockless { return Fake(), nil } - if common.TraceFlag { + if common.TraceFlag() { defer common.Stopwatch("LOCKER: Got lock on %v in", filename).Report() } common.Trace("LOCKER: Want lock on: %v", filename) - _, err := EnsureParentDirectory(filename) + if sharedLocation { + _, err := EnsureSharedParentDirectory(filename) + if err != nil { + return nil, err + } + } else { + _, err := EnsureParentDirectory(filename) + if err != nil { + return nil, err + } + } + file, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o666) if err != nil { return nil, err } - file, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600) + _, err = shared.MakeSharedFile(filename) if err != nil { return nil, err } @@ -29,12 +41,16 @@ func Locker(filename string, trycount int) (Releaser, error) { if err != nil { return nil, err } - return &Locked{file}, nil + lockpid := LockpidFor(filename) + latch := lockpid.Keepalive() + common.Trace("LOCKER: make marker %v", lockpid.Location()) + return &Locked{file, latch}, nil } func (it Locked) Release() error { defer it.Close() err := syscall.Flock(int(it.Fd()), int(syscall.LOCK_UN)) - common.Trace("LOCKER: release with err: %v", err) + common.Trace("LOCKER: release %v with err: %v", it.Name(), err) + close(it.Latch) return err } diff --git a/pathlib/lock_windows.go b/pathlib/lock_windows.go index 1990b940..a101dce7 100644 --- a/pathlib/lock_windows.go +++ b/pathlib/lock_windows.go @@ -1,3 +1,4 @@ +//go:build windows // +build windows package pathlib @@ -27,25 +28,32 @@ type filehandle interface { Fd() uintptr } -func Locker(filename string, trycount int) (Releaser, error) { - if Lockless { +func Locker(filename string, trycount int, sharedLocation bool) (Releaser, error) { + if common.WarrantyVoided() || Lockless { return Fake(), nil } var file *os.File var err error - if common.TraceFlag { + if common.TraceFlag() { defer func() { common.Stopwatch("LOCKER: Leaving lock on %v with %v retries left in", filename, trycount).Report() }() } common.Trace("LOCKER: Want lock on: %v", filename) - _, err = EnsureParentDirectory(filename) - if err != nil { - return nil, err + if sharedLocation { + _, err = EnsureSharedParentDirectory(filename) + if err != nil { + return nil, err + } + } else { + _, err := EnsureParentDirectory(filename) + if err != nil { + return nil, err + } } for { trycount -= 1 - file, err = os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600) + file, err = os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o666) if err != nil && trycount < 0 { return nil, err } @@ -53,6 +61,10 @@ func Locker(filename string, trycount int) (Releaser, error) { time.Sleep(40 * time.Millisecond) continue } + _, err = shared.MakeSharedFile(filename) + if err != nil { + return nil, err + } break } for { @@ -62,7 +74,10 @@ func Locker(filename string, trycount int) (Releaser, error) { return nil, err } if success { - return &Locked{file}, nil + lockpid := LockpidFor(filename) + latch := lockpid.Keepalive() + common.Trace("LOCKER: make marker %v", lockpid.Location()) + return &Locked{file, latch}, nil } time.Sleep(40 * time.Millisecond) } @@ -70,7 +85,8 @@ func Locker(filename string, trycount int) (Releaser, error) { func (it Locked) Release() error { success, err := trylock(unlockFile, it) - common.Trace("LOCKER: release success: %v with err: %v", success, err) + common.Trace("LOCKER: release %v success: %v with err: %v", it.Name(), success, err) + close(it.Latch) return err } diff --git a/pathlib/lockpids.go b/pathlib/lockpids.go new file mode 100644 index 00000000..34d48e8c --- /dev/null +++ b/pathlib/lockpids.go @@ -0,0 +1,217 @@ +package pathlib + +import ( + "fmt" + "os" + "os/user" + "path/filepath" + "regexp" + "runtime" + "strconv" + "strings" + "time" + + "github.com/robocorp/rcc/anywork" + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/fail" +) + +const ( + touchDelay = 7 + deadlineDelay = touchDelay * -5 + partSeparator = `___` +) + +var ( + slashPattern = regexp.MustCompile("[/\\\\]+") + underscorePattern = regexp.MustCompile("_+") + spacePattern = regexp.MustCompile("\\s+") +) + +type ( + Lockpid struct { + ParentID int + ProcessID int + Controller string + Space string + Username string + Basename string + } + Lockpids []*Lockpid +) + +func LockHoldersBy(filename string) (result Lockpids, err error) { + defer fail.Around(&err) + + holders, err := LoadLockpids() + fail.On(err != nil, "%v", err) + total := len(holders) + if total == 0 { + return holders, nil + } + + selector := unify(filepath.Base(filename)) + result = Lockpids{} + for _, candidate := range holders { + if candidate.Basename == selector { + result = append(result, candidate) + } + } + + return result, nil +} + +func LoadLockpids() (result Lockpids, err error) { + defer fail.Around(&err) + + deadline := time.Now().Add(deadlineDelay * time.Second) + result = make(Lockpids, 0, 10) + root := common.HololibPids() + entries, err := os.ReadDir(root) + fail.On(err != nil, "Failed to read lock pids directory, reason: %v", err) + +browsing: + for _, entry := range entries { + fullpath := filepath.Join(root, entry.Name()) + info, err := entry.Info() + if err != nil || info == nil { + continue + } + if info.IsDir() { + anywork.Backlog(func() { + TryRemoveAll("lockpid/dir", fullpath) + common.Trace(">> Trying to remove extra dir at lockpids: %q", fullpath) + }) + continue browsing + } + if info.ModTime().Before(deadline) { + anywork.Backlog(func() { + TryRemove("lockpid/stale", fullpath) + common.Trace(">> Trying to remove old file at lockpids: %q", fullpath) + }) + continue browsing + } + lockpid, ok := parseLockpid(entry.Name()) + if !ok { + anywork.Backlog(func() { + TryRemove("lockpid/unknown", fullpath) + common.Trace(">> Trying to remove unknown file at lockpids: %q", fullpath) + }) + continue browsing + } + result = append(result, lockpid) + } + return result, nil +} + +func parseLockpid(basename string) (*Lockpid, bool) { + parts := strings.Split(basename, partSeparator) + if len(parts) != 6 { + return nil, false + } + parentID, err := strconv.Atoi(parts[1]) + if err != nil { + return nil, false + } + processID, err := strconv.Atoi(parts[3]) + if err != nil { + return nil, false + } + return &Lockpid{ + ParentID: parentID, + ProcessID: processID, + Controller: unify(parts[0]), + Space: unify(parts[2]), + Username: unify(parts[4]), + Basename: unify(parts[5]), + }, true +} + +func LockpidFor(filename string) *Lockpid { + basename := filepath.Base(filename) + username := "anonymous" + who, err := user.Current() + if err == nil { + username = unslash(who.Username) + } + return &Lockpid{ + ParentID: os.Getppid(), + ProcessID: os.Getpid(), + Controller: unify(common.ControllerType), + Space: unify(common.HolotreeSpace), + Username: unify(username), + Basename: unify(basename), + } +} + +func (it *Lockpid) Message() string { + return fmt.Sprintf("Possibly pending lock %q, user: %q, space: %q, and controller: %q (parent/pid: %d/%d). May cause environment wait/build delay.", it.Basename, it.Username, it.Space, it.Controller, it.ParentID, it.ProcessID) +} + +func (it *Lockpid) Keepalive() chan bool { + latch := make(chan bool) + go keepFresh(it, latch) + runtime.Gosched() + common.Trace("Trying to keep lockpid %q fresh fron now on.", it.Location()) + return latch +} + +func (it *Lockpid) Touch() { + where := it.Location() + anywork.Backlog(func() { + ForceTouchWhen(where, time.Now()) + common.Trace(">> Tried to touch lockpid %q now.", where) + }) + runtime.Gosched() +} + +func (it *Lockpid) Erase() { + where := it.Location() + anywork.Backlog(func() { + TryRemove("lockpid", where) + common.Trace(">> Tried to erase lockpid %q now.", where) + }) + runtime.Gosched() +} + +func (it *Lockpid) Filename() string { + return fmt.Sprintf("%s___%d___%s___%d___%s___%s", it.Controller, it.ParentID, it.Space, it.ProcessID, it.Username, it.Basename) +} + +func (it *Lockpid) Location() string { + return filepath.Join(common.HololibPids(), it.Filename()) +} + +func keepFresh(lockpid *Lockpid, latch chan bool) { + defer lockpid.Erase() + delay := touchDelay * time.Second +forever: + for { + lockpid.Touch() + select { + case <-latch: + break forever + case <-time.After(delay): + continue forever + } + } +} + +func unspace(text string) string { + parts := spacePattern.Split(text, -1) + return strings.Join(parts, "_") +} + +func unslash(text string) string { + parts := slashPattern.Split(text, -1) + return strings.Join(parts, "_") +} + +func oneunderscore(text string) string { + parts := underscorePattern.Split(text, -1) + return strings.Join(parts, "_") +} + +func unify(text string) string { + return oneunderscore(unslash(unspace(strings.TrimSpace(text)))) +} diff --git a/pathlib/sha256_test.go b/pathlib/sha256_test.go index 89df74a8..98c63542 100644 --- a/pathlib/sha256_test.go +++ b/pathlib/sha256_test.go @@ -1,6 +1,8 @@ package pathlib_test import ( + "bytes" + "os" "testing" "github.com/robocorp/rcc/hamlet" @@ -18,6 +20,11 @@ func TestCalculateSha256OfFiles(t *testing.T) { must.Nil(err) must.Equal("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", digest) + contents, err := os.ReadFile("testdata/hello.txt") + must.Nil(err) + // check that the file has no CR characters + must.Equal(-1, bytes.Index(contents, []byte("\r"))) + digest, err = pathlib.Sha256("testdata/hello.txt") must.Nil(err) must.Equal("d9014c4624844aa5bac314773d6b689ad467fa4e1d1a50a1b8a99d5a95f72ff5", digest) diff --git a/pathlib/shared.go b/pathlib/shared.go new file mode 100644 index 00000000..6dfa2067 --- /dev/null +++ b/pathlib/shared.go @@ -0,0 +1,35 @@ +package pathlib + +import ( + "os" + + "github.com/robocorp/rcc/fail" +) + +type ( + Shared interface { + MakeSharedFile(fullpath string) (string, error) + MakeSharedDir(fullpath string) (string, error) + } + + privateSetup uint8 + sharedSetup uint8 +) + +func (it privateSetup) MakeSharedFile(fullpath string) (string, error) { + return fullpath, nil +} + +func (it privateSetup) MakeSharedDir(fullpath string) (string, error) { + return makeModedDir(fullpath, 0750) +} + +func (it sharedSetup) MakeSharedFile(fullpath string) (string, error) { + stat, err := os.Stat(fullpath) + fail.On(err != nil, "Failed to stat file %q, reason: %v", fullpath, err) + return ensureCorrectMode(fullpath, stat, 0666) +} + +func (it sharedSetup) MakeSharedDir(fullpath string) (string, error) { + return makeModedDir(fullpath, 0777) +} diff --git a/pathlib/targetpath.go b/pathlib/targetpath.go index 69bc7fb2..8ed01729 100644 --- a/pathlib/targetpath.go +++ b/pathlib/targetpath.go @@ -4,13 +4,54 @@ import ( "fmt" "os" "path/filepath" + "regexp" "strings" + + "github.com/robocorp/rcc/common" ) type PathParts []string +func noDuplicates(paths PathParts) PathParts { + seen := make(map[string]bool) + result := make(PathParts, 0, len(paths)) + for _, part := range paths { + if seen[part] { + continue + } + result = append(result, part) + seen[part] = true + } + return result +} + +func noPreviousHolotrees(paths PathParts) PathParts { + form := fmt.Sprintf("\\b(?:%s|UNMNGED)_[0-9a-f]{7}_[0-9a-f]{8}\\b", common.SymbolicUserIdentity()) + pattern, err := regexp.Compile(form) + if err != nil { + return paths + } + result := make(PathParts, 0, len(paths)) + for _, part := range paths { + if !pattern.MatchString(part) { + result = append(result, part) + } + } + return result +} + func TargetPath() PathParts { - return filepath.SplitList(os.Getenv("PATH")) + return noPreviousHolotrees(noDuplicates(filepath.SplitList(os.Getenv("PATH")))) +} + +func EnvironmentPath(environment []string) PathParts { + path := "" + for _, entry := range environment { + if strings.HasPrefix(strings.ToLower(entry), "path=") { + path = entry[5:] + } + } + return noDuplicates(filepath.SplitList(path)) } func PathFrom(parts ...string) PathParts { diff --git a/pathlib/testdata/commented_ignores b/pathlib/testdata/commented_ignores new file mode 100644 index 00000000..5023e99a --- /dev/null +++ b/pathlib/testdata/commented_ignores @@ -0,0 +1,4 @@ +# this is empty ignore file, no patterns + +# so this should generate error when loaded, even when there is empty lines +# and comments on it diff --git a/pathlib/testdata/valid_ignores b/pathlib/testdata/valid_ignores new file mode 100644 index 00000000..397b4a76 --- /dev/null +++ b/pathlib/testdata/valid_ignores @@ -0,0 +1 @@ +*.log diff --git a/pathlib/touch.go b/pathlib/touch.go index ca7017dc..7c965b21 100644 --- a/pathlib/touch.go +++ b/pathlib/touch.go @@ -3,8 +3,23 @@ package pathlib import ( "os" "time" + + "github.com/robocorp/rcc/common" ) func TouchWhen(location string, when time.Time) { - os.Chtimes(location, when, when) + err := os.Chtimes(location, when, when) + if err != nil { + common.Debug("Touching file %q failed, reason: %v ... ignored!", location, err) + } +} + +func ForceTouchWhen(location string, when time.Time) { + if !Exists(location) { + err := WriteFile(location, []byte{}, 0o644) + if err != nil { + common.Debug("Touch/creating file %q failed, reason: %v ... ignored!", location, err) + } + } + TouchWhen(location, when) } diff --git a/pathlib/validators.go b/pathlib/validators.go index ebaff419..f9f8929e 100644 --- a/pathlib/validators.go +++ b/pathlib/validators.go @@ -1,9 +1,10 @@ package pathlib import ( - "errors" "fmt" "os" + + "github.com/robocorp/rcc/pretty" ) func FileExist(name string) bool { @@ -21,13 +22,42 @@ func EnsureDirectoryExists(directory string) error { func EnsureEmptyDirectory(directory string) error { fullpath, err := EnsureDirectory(directory) - handle, err := os.Open(fullpath) if err != nil { return err } - entries, err := handle.Readdir(-1) + entries, err := os.ReadDir(fullpath) + if err != nil { + return err + } if len(entries) > 0 { - return errors.New(fmt.Sprintf("Directory %s is not empty!", fullpath)) + return fmt.Errorf("Directory %s is not empty!", fullpath) } return nil } + +func NoteDirectoryContent(context, directory string, guide bool) { + if !IsDir(directory) { + return + } + fullpath, err := Abs(directory) + if err != nil { + return + } + entries, err := os.ReadDir(fullpath) + if err != nil { + return + } + noted := false + for _, entry := range entries { + if entry.Name() != "journal.run" { + pretty.Note("%s %q already has %q in it.", context, fullpath, entry.Name()) + noted = true + } + } + if guide && noted { + pretty.Highlight("Above notes mean, that there were files present in directory that was supposed to be empty!") + pretty.Highlight("In robot development phase, it might be ok to have these files while building robot.") + pretty.Highlight("In production robot/assistant, this might be a mistake, where development files were") + pretty.Highlight("left inside robot.zip file. Report these to developer who made this robot/assistant.") + } +} diff --git a/pathlib/variables.go b/pathlib/variables.go index 0f6b7771..71360728 100644 --- a/pathlib/variables.go +++ b/pathlib/variables.go @@ -1,5 +1,20 @@ package pathlib +import "github.com/robocorp/rcc/common" + var ( Lockless bool + shared Shared ) + +func init() { + if common.SharedHolotree { + ForceShared() + } else { + shared = privateSetup(1) + } +} + +func ForceShared() { + shared = sharedSetup(9) +} diff --git a/pathlib/walk.go b/pathlib/walk.go index 9bedd98a..a435214e 100644 --- a/pathlib/walk.go +++ b/pathlib/walk.go @@ -1,14 +1,17 @@ package pathlib import ( - "io/ioutil" + "fmt" "os" "path/filepath" "sort" "strings" "time" + + "github.com/robocorp/rcc/pretty" ) +type Forced func(os.FileInfo) bool type Ignore func(os.FileInfo) bool type Report func(string, string, os.FileInfo) @@ -32,6 +35,16 @@ func IgnoreDirectories(target os.FileInfo) bool { return target.IsDir() } +func ForceNothing(_ os.FileInfo) bool { + return false +} + +func ForceFilename(filename string) Forced { + return func(target os.FileInfo) bool { + return !target.IsDir() && target.Name() == filename + } +} + func NoReporting(string, string, os.FileInfo) { } @@ -81,8 +94,8 @@ func IgnorePattern(text string) Ignore { return CompositeIgnore(exactIgnore(text).Ignore, globIgnore(text).Ignore) } -func LoadIgnoreFile(filename string) (Ignore, error) { - content, err := ioutil.ReadFile(filename) +func LoadIgnoreFile(filename string, strict bool) (Ignore, error) { + content, err := os.ReadFile(filename) if err != nil { return nil, err } @@ -94,13 +107,20 @@ func LoadIgnoreFile(filename string) (Ignore, error) { } result = append(result, IgnorePattern(line)) } + if strict && len(result) == 0 { + return nil, fmt.Errorf("Ignore file %q has no valid patterns in it!", filename) + } + if len(result) == 0 { + pretty.Warning("Ignore file %q has no valid patterns in it!", filename) + return IgnoreNothing, nil + } return CompositeIgnore(result...), nil } func LoadIgnoreFiles(filenames []string) (Ignore, error) { result := make([]Ignore, 0, len(filenames)) for _, filename := range filenames { - ignore, err := LoadIgnoreFile(filename) + ignore, err := LoadIgnoreFile(filename, false) if err != nil { return nil, err } @@ -122,20 +142,38 @@ func folderEntries(directory string) ([]os.FileInfo, error) { return entries, nil } -func recursiveWalk(directory, prefix string, ignore Ignore, report Report) error { +func recursiveDirWalk(here os.FileInfo, directory, prefix string, report Report) error { + entries, err := folderEntries(directory) + if err != nil { + return err + } + sorted(entries) + for _, entry := range entries { + if !entry.IsDir() { + continue + } + nextPrefix := filepath.Join(prefix, entry.Name()) + entryPath := filepath.Join(directory, entry.Name()) + recursiveDirWalk(entry, entryPath, nextPrefix, report) + } + report(directory, prefix, here) + return nil +} + +func recursiveWalk(directory, prefix string, force Forced, ignore Ignore, report Report) error { entries, err := folderEntries(directory) if err != nil { return err } sorted(entries) for _, entry := range entries { - if ignore(entry) { + if !force(entry) && ignore(entry) { continue } nextPrefix := filepath.Join(prefix, entry.Name()) entryPath := filepath.Join(directory, entry.Name()) if entry.IsDir() { - recursiveWalk(entryPath, nextPrefix, ignore, report) + recursiveWalk(entryPath, nextPrefix, force, ignore, report) } else { report(entryPath, nextPrefix, entry) } @@ -143,10 +181,39 @@ func recursiveWalk(directory, prefix string, ignore Ignore, report Report) error return nil } -func Walk(directory string, ignore Ignore, report Report) error { +func ForceWalk(directory string, force Forced, ignore Ignore, report Report) error { + fullpath, err := filepath.Abs(directory) + if err != nil { + return err + } + return recursiveWalk(fullpath, ".", force, ignore, report) +} + +func DirWalk(directory string, report Report) error { fullpath, err := filepath.Abs(directory) if err != nil { return err } - return recursiveWalk(fullpath, ".", ignore, report) + entry, err := os.Stat(fullpath) + if err != nil { + return err + } + return recursiveDirWalk(entry, fullpath, ".", report) +} + +func Walk(directory string, ignore Ignore, report Report) error { + return ForceWalk(directory, ForceNothing, ignore, report) +} + +func RecursiveGlob(directory string, pattern string) []string { + result := []string{} + ignore := func(entry os.FileInfo) bool { + match, err := filepath.Match(pattern, entry.Name()) + return err != nil || !entry.IsDir() && !match + } + capture := func(_, localpath string, _ os.FileInfo) { + result = append(result, localpath) + } + ForceWalk(directory, ForceNothing, ignore, capture) + return result } diff --git a/pathlib/walk_test.go b/pathlib/walk_test.go index 76834ad5..9c391e9e 100644 --- a/pathlib/walk_test.go +++ b/pathlib/walk_test.go @@ -83,7 +83,7 @@ func TestUseCompositeIgnorePattern(t *testing.T) { func TestCanLoadIgnoreFile(t *testing.T) { must_be, wont_be := hamlet.Specifications(t) - sut, err := pathlib.LoadIgnoreFile("testdata/missing") + sut, err := pathlib.LoadIgnoreFile("testdata/missing", true) must_be.Nil(sut) wont_be.Nil(err) } @@ -91,7 +91,23 @@ func TestCanLoadIgnoreFile(t *testing.T) { func TestCanLoadEmptyIgnoreFile(t *testing.T) { must_be, wont_be := hamlet.Specifications(t) - sut, err := pathlib.LoadIgnoreFile("testdata/empty") + sut, err := pathlib.LoadIgnoreFile("testdata/empty", true) + must_be.Nil(sut) + wont_be.Nil(err) +} + +func TestCanLoadCommentOnlyIgnoreFile(t *testing.T) { + must_be, wont_be := hamlet.Specifications(t) + + sut, err := pathlib.LoadIgnoreFile("testdata/commented_ignores", true) + must_be.Nil(sut) + wont_be.Nil(err) +} + +func TestCanLoadValidIgnoreFile(t *testing.T) { + must_be, wont_be := hamlet.Specifications(t) + + sut, err := pathlib.LoadIgnoreFile("testdata/valid_ignores", true) wont_be.Nil(sut) must_be.Nil(err) } diff --git a/pretty/functions.go b/pretty/functions.go index 2441f870..f1229fdc 100644 --- a/pretty/functions.go +++ b/pretty/functions.go @@ -2,15 +2,64 @@ package pretty import ( "fmt" + "strings" + "time" "github.com/robocorp/rcc/common" ) +const ( + rccpov = `From rcc %q (controller: %q) point of view, %q was` + maxSteps = 15 +) + +var ( + ProgressMark time.Time + onlyOnceMessages = make(map[string]bool) +) + +func init() { + ProgressMark = time.Now() +} + func Ok() error { common.Log("%sOK.%s", Green, Reset) return nil } +func JustOnce(format string, rest ...interface{}) { + message := fmt.Sprintf(format, rest...) + if !onlyOnceMessages[message] { + onlyOnceMessages[message] = true + Highlight(format, rest...) + } +} + +func DebugNote(format string, rest ...interface{}) { + niceform := fmt.Sprintf("%s%sNote: %s%s", Blue, Bold, format, Reset) + common.Debug(niceform, rest...) +} + +func Note(format string, rest ...interface{}) { + niceform := fmt.Sprintf("%s%sNote: %s%s", Cyan, Bold, format, Reset) + common.Log(niceform, rest...) +} + +func Warning(format string, rest ...interface{}) { + niceform := fmt.Sprintf("%sWarning: %s%s", Yellow, format, Reset) + common.Log(niceform, rest...) +} + +func Highlight(format string, rest ...interface{}) { + niceform := fmt.Sprintf("%s%s%s", Magenta, format, Reset) + common.Log(niceform, rest...) +} + +func Lowlight(format string, rest ...interface{}) { + niceform := fmt.Sprintf("%s%s%s", Grey, format, Reset) + common.Log(niceform, rest...) +} + func Exit(code int, format string, rest ...interface{}) { var niceform string if code == 0 { @@ -20,3 +69,49 @@ func Exit(code int, format string, rest ...interface{}) { } common.Exit(code, niceform, rest...) } + +// Guard watches, that only truthful shall pass. Otherwise exits with code and details. +func Guard(truth bool, code int, format string, rest ...interface{}) { + if !truth { + Exit(code, format, rest...) + } +} + +func RccPointOfView(context string, err error) { + explain := fmt.Sprintf(rccpov, common.Version, common.ControllerType, context) + printer := Lowlight + message := fmt.Sprintf("@@@ %s SUCCESS. @@@", explain) + journal := fmt.Sprintf("%s SUCCESS.", explain) + if err != nil { + printer = Highlight + message = fmt.Sprintf("@@@ %s FAILURE, reason: %q. See details above. @@@", explain, err) + journal = fmt.Sprintf("%s FAILURE, reason: %s", explain, err) + } + banner := strings.Repeat("@", len(message)) + printer(banner) + printer(message) + printer(banner) + common.RunJournal("robot exit", journal, "rcc point of view") +} + +func Regression(step int, form string, details ...interface{}) { + progress(Red, step, form, details...) +} + +func Progress(step int, form string, details ...interface{}) { + color := Cyan + if step == maxSteps { + color = Green + } + progress(color, step, form, details...) +} + +func progress(color string, step int, form string, details ...interface{}) { + previous := ProgressMark + ProgressMark = time.Now() + delta := ProgressMark.Sub(previous).Round(1 * time.Millisecond).Seconds() + message := fmt.Sprintf(form, details...) + common.Log("%s#### Progress: %02d/%d %s %8.3fs %s%s", color, step, maxSteps, common.Version, delta, message, Reset) + common.Timeline("%d/%d %s", step, maxSteps, message) + common.RunJournal("environment", "build", "Progress: %02d/%d %s %8.3fs %s", step, maxSteps, common.Version, delta, message) +} diff --git a/pretty/pager.go b/pretty/pager.go new file mode 100644 index 00000000..69be71cb --- /dev/null +++ b/pretty/pager.go @@ -0,0 +1,59 @@ +package pretty + +import ( + "bufio" + "fmt" + "os" + "regexp" + "strings" + + "github.com/robocorp/rcc/common" + "golang.org/x/term" +) + +var ( + titlePattern = regexp.MustCompile("^#{1,5}\\s+") + codePattern = regexp.MustCompile("^ {4,}\\S+") + blockPattern = regexp.MustCompile("^```") +) + +func Page(content []byte) { + width, height, err := term.GetSize(int(os.Stdout.Fd())) + if err != nil || !Interactive { + common.Stdout("\n%s\n", content) + return + } + + titleStyle := fmt.Sprintf("%s%s", Bold, Underline) + codeStyle := Faint + + limit := height - 3 + reader := bufio.NewReader(os.Stdin) + lines := strings.SplitAfter(string(content), "\n") + fmt.Printf("%s%s", Home, Clear) + row := 0 + block := false + for _, line := range lines { + flat := strings.TrimRight(line, " \t\r\n") + if blockPattern.MatchString(flat) { + block = !block + continue + } + adjust := len(flat) / width + row += 1 + adjust + if row > limit { + fmt.Print("\n-- press enter to continue or ctrl-c to stop --") + reader.ReadLine() + fmt.Printf("%s%s", Home, Clear) + row = 1 + } + style := "" + if titlePattern.MatchString(flat) { + style = titleStyle + } + if block || codePattern.MatchString(flat) { + style = codeStyle + } + fmt.Printf("%s%s%s\n", style, flat, Reset) + } +} diff --git a/pretty/pretty_test.go b/pretty/pretty_test.go new file mode 100644 index 00000000..69a3f3cf --- /dev/null +++ b/pretty/pretty_test.go @@ -0,0 +1 @@ +package pretty_test diff --git a/pretty/setup_darwin.go b/pretty/setup_darwin.go index e92ce1bb..3f681537 100644 --- a/pretty/setup_darwin.go +++ b/pretty/setup_darwin.go @@ -1,6 +1,6 @@ package pretty -func localSetup() { +func localSetup(bool) { Iconic = true Disabled = false } diff --git a/pretty/setup_linux.go b/pretty/setup_linux.go index f96a43c2..f3b373db 100644 --- a/pretty/setup_linux.go +++ b/pretty/setup_linux.go @@ -1,6 +1,6 @@ package pretty -func localSetup() { +func localSetup(bool) { Iconic = false Disabled = false } diff --git a/pretty/setup_windows.go b/pretty/setup_windows.go index 02445cf2..5ce5d68b 100644 --- a/pretty/setup_windows.go +++ b/pretty/setup_windows.go @@ -1,3 +1,4 @@ +//go:build windows || !darwin || !linux // +build windows !darwin !linux package pretty @@ -12,29 +13,32 @@ const ( ENABLE_VIRTUAL_TERMINAL_PROCESSING = 0x4 ) -func localSetup() { +func localSetup(interactive bool) { Iconic = false Disabled = true + if !interactive { + return + } kernel32 := syscall.NewLazyDLL("kernel32.dll") if kernel32 == nil { - common.Trace("Cannot use colors. Did not get kernel32.dll!") + common.Trace("Error: Cannot use colors. Did not get kernel32.dll!") return } setConsoleMode := kernel32.NewProc("SetConsoleMode") if setConsoleMode == nil { - common.Trace("Cannot use colors. Did not get SetConsoleMode!") + common.Trace("Error: Cannot use colors. Did not get SetConsoleMode!") return } target := syscall.Stdout var mode uint32 err := syscall.GetConsoleMode(target, &mode) if err != nil { - common.Trace("Cannot use colors. Got mode error '%v'!", err) + common.Trace("Error: Cannot use colors. Got mode error '%v'!", err) } mode |= ENABLE_VIRTUAL_TERMINAL_PROCESSING success, _, err := setConsoleMode.Call(uintptr(target), uintptr(mode)) Disabled = success == 0 if Disabled && err != nil { - common.Trace("Cannot use colors. Got error '%v'!", err) + common.Trace("Error: Cannot use colors. Got error '%v'!", err) } } diff --git a/pretty/variables.go b/pretty/variables.go index 6f661bfe..938e2952 100644 --- a/pretty/variables.go +++ b/pretty/variables.go @@ -8,6 +8,7 @@ import ( ) var ( + Colorless bool Iconic bool Disabled bool Interactive bool @@ -16,11 +17,19 @@ var ( Black string Red string Green string + Blue string Yellow string + Magenta string Cyan string Reset string Sparkles string Rocket string + Home string + Clear string + Bold string + Faint string + Italic string + Underline string ) func Setup() { @@ -29,20 +38,28 @@ func Setup() { stderr := isatty.IsTerminal(os.Stderr.Fd()) Interactive = stdin && stdout && stderr - localSetup() + localSetup(Interactive) common.Trace("Interactive mode enabled: %v; colors enabled: %v; icons enabled: %v", Interactive, !Disabled, Iconic) - if Interactive && !Disabled { + if Interactive && !Disabled && !Colorless { White = csi("97m") Grey = csi("90m") Black = csi("30m") Red = csi("91m") Green = csi("92m") - Cyan = csi("96m") Yellow = csi("93m") + Blue = csi("94m") + Magenta = csi("95m") + Cyan = csi("96m") Reset = csi("0m") + Home = csi("1;1H") + Clear = csi("0J") + Bold = csi("1m") + Faint = csi("2m") + Italic = csi("3m") + Underline = csi("4m") } - if Iconic { + if Iconic && !Colorless { Sparkles = "\u2728 " Rocket = "\U0001F680 " } diff --git a/remotree/delta.go b/remotree/delta.go new file mode 100644 index 00000000..22cb799e --- /dev/null +++ b/remotree/delta.go @@ -0,0 +1,147 @@ +package remotree + +import ( + "archive/zip" + "bufio" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" + + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/fail" + "github.com/robocorp/rcc/htfs" + "github.com/robocorp/rcc/operations" + "github.com/robocorp/rcc/pathlib" + "github.com/robocorp/rcc/set" +) + +func isSelfRequest(request *http.Request) bool { + identity, ok := request.Header[operations.X_RCC_RANDOM_IDENTITY] + return ok && len(identity) > 0 && identity[0] == common.RandomIdentifier() +} + +func makeDeltaHandler(queries Partqueries) http.HandlerFunc { + return func(response http.ResponseWriter, request *http.Request) { + catalog := filepath.Base(request.URL.Path) + defer common.Stopwatch("Delta of catalog %q took", catalog).Debug() + if request.Method != http.MethodPost { + response.WriteHeader(http.StatusMethodNotAllowed) + common.Trace("Delta: rejecting request %q for catalog %q.", request.Method, catalog) + return + } + if isSelfRequest(request) { + response.WriteHeader(http.StatusConflict) + common.Trace("Delta: rejecting /SELF/ request for catalog %q.", catalog) + return + } + reply := make(chan string) + queries <- &Partquery{ + Catalog: catalog, + Reply: reply, + } + known, ok := <-reply + common.Debug("query handler: %q -> %v", catalog, ok) + if !ok { + response.WriteHeader(http.StatusNotFound) + response.Write([]byte("404 not found, sorry")) + return + } + + membership := set.Membership(strings.Split(known, "\n")) + + approved := make([]string, 0, 1000) + todo := bufio.NewReader(request.Body) + todoloop: + for { + line, err := todo.ReadString('\n') + stopping := err == io.EOF + candidate := filepath.Base(strings.TrimSpace(line)) + if len(candidate) > 10 { + if membership[candidate] { + approved = append(approved, candidate) + } else { + common.Trace("DELTA: ignoring extra %q entry, not part of set!", candidate) + if !stopping { + continue todoloop + } + } + } + if stopping { + break todoloop + } + if err != nil { + common.Trace("DELTA: error %v with line %q", err, line) + break todoloop + } + } + + partfile, err := exportMissing(catalog, approved) + if err != nil { + common.Debug("DELTA: error %v", err) + response.WriteHeader(http.StatusInternalServerError) + return + } + + http.ServeFile(response, request, partfile) + } +} + +func tempDir() (string, bool) { + root := pathlib.TempDir() + directory := filepath.Join(root, "rccremote") + fullpath, err := pathlib.EnsureDirectory(directory) + if err != nil { + return root, false + } + return fullpath, true +} + +func exportMissing(catalog string, missing []string) (result string, err error) { + defer fail.Around(&err) + + tempdir, _ := tempDir() + identity := common.Digest(strings.Join(missing, "\n")) + filename := filepath.Join(tempdir, fmt.Sprintf("%s_parts.zip", identity)) + if pathlib.IsFile(filename) { + common.Debug("Using existing cache file %q [size: %s]", filename, pathlib.HumaneSize(filename)) + return filename, nil + } + + tempfile := filepath.Join(tempdir, fmt.Sprintf("%s_%x_build.zip", identity, os.Getppid())) + err = exportMissingToFile(catalog, missing, tempfile) + fail.On(err != nil, "%v", err) + + err = os.Rename(tempfile, filename) + fail.On(err != nil, "%v", err) + + common.Debug("Created cache file %q [size: %s]", filename, pathlib.HumaneSize(filename)) + return filename, nil +} + +func exportMissingToFile(catalog string, missing []string, filename string) (err error) { + defer fail.Around(&err) + + handle, err := pathlib.Create(filename) + fail.On(err != nil, "Could not create export file %q, reason: %v", filename, err) + defer handle.Close() + + sink := zip.NewWriter(handle) + defer sink.Close() + + for _, member := range missing { + relative := htfs.RelativeDefaultLocation(member) + fullpath := htfs.ExactDefaultLocation(member) + err = operations.ZipAppend(sink, fullpath, relative) + fail.On(err != nil, "Could not zip file %q, reason: %v", fullpath, err) + } + + fullpath := filepath.Join(common.HololibCatalogLocation(), catalog) + relative, err := filepath.Rel(common.HololibLocation(), fullpath) + fail.On(err != nil, "Could not get relative path for catalog %q, reason: %v", fullpath, err) + err = operations.ZipAppend(sink, fullpath, relative) + fail.On(err != nil, "Could not zip catalog %q, reason: %v", fullpath, err) + return nil +} diff --git a/remotree/listings.go b/remotree/listings.go new file mode 100644 index 00000000..018e1203 --- /dev/null +++ b/remotree/listings.go @@ -0,0 +1,114 @@ +package remotree + +import ( + "bufio" + "net/http" + "path/filepath" + "strings" + + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/fail" + "github.com/robocorp/rcc/htfs" + "github.com/robocorp/rcc/set" +) + +const ( + partCacheSize = 20 +) + +func makeQueryHandler(queries Partqueries, triggers chan string) http.HandlerFunc { + return func(response http.ResponseWriter, request *http.Request) { + catalog := filepath.Base(request.URL.Path) + defer common.Stopwatch("Query of catalog %q took", catalog).Debug() + if request.Method != http.MethodGet { + response.WriteHeader(http.StatusMethodNotAllowed) + common.Trace("Query: rejecting request %q for catalog %q.", request.Method, catalog) + return + } + if isSelfRequest(request) { + response.WriteHeader(http.StatusConflict) + common.Trace("Query: rejecting /SELF/ request for catalog %q.", catalog) + return + } + reply := make(chan string) + queries <- &Partquery{ + Catalog: catalog, + Reply: reply, + } + content, ok := <-reply + common.Debug("query handler: %q -> %v", catalog, ok) + if !ok { + triggers <- catalog + response.WriteHeader(http.StatusNotFound) + response.Write([]byte("404 not found, sorry")) + return + } + headers := response.Header() + headers.Add("Content-Type", "text/plain") + response.WriteHeader(http.StatusOK) + writer := bufio.NewWriter(response) + defer writer.Flush() + writer.WriteString(content) + } +} + +func loadSingleCatalog(catalog string) (root *htfs.Root, err error) { + defer fail.Around(&err) + tempdir := filepath.Join(common.ProductTemp(), "rccremote") + shadow, err := htfs.NewRoot(tempdir) + fail.On(err != nil, "Could not create root, reason: %v", err) + filename := filepath.Join(common.HololibCatalogLocation(), catalog) + err = shadow.LoadFrom(filename) + fail.On(err != nil, "Could not load root, reason: %v", err) + common.Trace("Catalog %q loaded.", catalog) + return shadow, nil +} + +func loadCatalogParts(catalog string) (string, bool) { + catalogs := htfs.CatalogNames() + if !set.Member(catalogs, catalog) { + return "", false + } + root, err := loadSingleCatalog(catalog) + if err != nil { + return "", false + } + collector := make(map[string]string) + task := htfs.DigestMapper(collector) + err = task(root.Path, root.Tree) + if err != nil { + return "", false + } + keys := set.Keys(collector) + return strings.Join(keys, "\n"), true +} + +func listProvider(queries Partqueries) { + cache := make(map[string]string) + keys := make([]string, partCacheSize) + cursor := uint64(0) +loop: + for { + query, ok := <-queries + if !ok { + break loop + } + known, ok := cache[query.Catalog] + if ok { + query.Reply <- known + close(query.Reply) + continue + } + created, ok := loadCatalogParts(query.Catalog) + if !ok { + close(query.Reply) + continue + } + delete(cache, keys[cursor%partCacheSize]) + cache[query.Catalog] = created + keys[cursor] = query.Catalog + cursor += 1 + query.Reply <- created + close(query.Reply) + } +} diff --git a/remotree/manage.go b/remotree/manage.go new file mode 100644 index 00000000..8764904c --- /dev/null +++ b/remotree/manage.go @@ -0,0 +1,26 @@ +package remotree + +import ( + "path/filepath" + + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/pathlib" +) + +func cleanupHoldStorage(storage string) error { + if !pathlib.IsDir(storage) { + return nil + } + filenames, err := filepath.Glob(filepath.Join(storage, "*.hld")) + if err != nil { + return err + } + for _, filename := range filenames { + err = pathlib.TryRemove("hold", filename) + if err != nil { + return err + } + common.Debug("Old hold file %q removed.", filename) + } + return nil +} diff --git a/remotree/messages.go b/remotree/messages.go new file mode 100644 index 00000000..de32f1cb --- /dev/null +++ b/remotree/messages.go @@ -0,0 +1,9 @@ +package remotree + +type ( + Partquery struct { + Catalog string + Reply chan string + } + Partqueries chan *Partquery +) diff --git a/remotree/missing_test.go b/remotree/missing_test.go new file mode 100644 index 00000000..3410dd67 --- /dev/null +++ b/remotree/missing_test.go @@ -0,0 +1 @@ +package remotree_test diff --git a/remotree/server.go b/remotree/server.go new file mode 100644 index 00000000..e72ba599 --- /dev/null +++ b/remotree/server.go @@ -0,0 +1,67 @@ +package remotree + +import ( + "context" + "fmt" + "net/http" + "os" + "os/signal" + "path/filepath" + "syscall" + "time" + + "github.com/robocorp/rcc/pathlib" +) + +func Serve(address string, port int, domain, storage string) error { + // we need + // - query handler (for just catalog hashes) + // - partial content sender (for sending delta catalog) + // - webserver + holding := filepath.Join(storage, "hold") + err := cleanupHoldStorage(holding) + if err != nil { + return err + } + defer cleanupHoldStorage(holding) + + tempdir, ok := tempDir() + if ok { + pathlib.TryRemoveAll("remotree.Serve[start]", tempdir) + defer pathlib.TryRemoveAll("remotree.Serve[defer]", tempdir) + } + + triggers := make(chan string, 20) + defer close(triggers) + + partqueries := make(Partqueries) + defer close(partqueries) + + go listProvider(partqueries) + go pullProcess(triggers) + + listen := fmt.Sprintf("%s:%d", address, port) + mux := http.NewServeMux() + server := &http.Server{ + Addr: listen, + Handler: mux, + ReadTimeout: 2 * time.Minute, + WriteTimeout: 30 * time.Minute, + MaxHeaderBytes: 1 << 14, + } + + mux.HandleFunc("/parts/", makeQueryHandler(partqueries, triggers)) + mux.HandleFunc("/delta/", makeDeltaHandler(partqueries)) + mux.HandleFunc("/force/", makeTriggerHandler(triggers)) + + go server.ListenAndServe() + + return runTillSignal(server) +} + +func runTillSignal(server *http.Server) error { + signals := make(chan os.Signal, 1) + signal.Notify(signals, syscall.SIGINT, syscall.SIGHUP, syscall.SIGTERM) + <-signals + return server.Shutdown(context.TODO()) +} diff --git a/remotree/trigger.go b/remotree/trigger.go new file mode 100644 index 00000000..3884fb15 --- /dev/null +++ b/remotree/trigger.go @@ -0,0 +1,51 @@ +package remotree + +import ( + "net/http" + "path/filepath" + + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/operations" + "github.com/robocorp/rcc/pretty" +) + +func makeTriggerHandler(requests chan string) http.HandlerFunc { + return func(response http.ResponseWriter, request *http.Request) { + catalog := filepath.Base(request.URL.Path) + defer common.Stopwatch("Trigger of catalog %q took", catalog).Debug() + requests <- catalog + } +} + +func pullOperation(counter int, catalog, remoteOrigin string) { + defer common.Stopwatch("#%d: pull opearation lasted", counter).Report() + common.Log("#%d: Trying to pull %q from %q ...", counter, catalog, remoteOrigin) + err := operations.PullCatalog(remoteOrigin, catalog, true) + if err != nil { + pretty.Warning("#%d: Failed to pull %q from %q, reason: %v", counter, catalog, remoteOrigin, err) + } else { + common.Log("#%d: Pull %q from %q completed.", counter, catalog, remoteOrigin) + } +} + +func pullProcess(requests chan string) { + remoteOrigin := common.RccRemoteOrigin() + disabled := len(remoteOrigin) == 0 + if disabled { + pretty.Note("Wont pull anything since RCC_REMOTE_ORIGIN is not defined.") + } + counter := 0 +forever: + for { + catalog, ok := <-requests + if !ok { + break forever + } + counter += 1 + if disabled { + pretty.Warning("Cannot #%d pull %q since RCC_REMOTE_ORIGIN is not defined.", counter, catalog) + continue + } + pullOperation(counter, catalog, remoteOrigin) + } +} diff --git a/robot/config.go b/robot/config.go deleted file mode 100644 index 3cf004d7..00000000 --- a/robot/config.go +++ /dev/null @@ -1,278 +0,0 @@ -package robot - -import ( - "errors" - "fmt" - "io/ioutil" - "os" - "path/filepath" - "sort" - "strings" - - "github.com/robocorp/rcc/conda" - "github.com/robocorp/rcc/pathlib" - - "gopkg.in/yaml.v2" -) - -type Config struct { - Activities map[string]*Activity `yaml:"activities"` - Conda string `yaml:"condaConfig"` - Ignored []string `yaml:"ignoreFiles"` - Root string -} - -type Activity struct { - Output string `yaml:"output"` - Root string `yaml:"activityRoot"` - Environment *Environment `yaml:"environment"` - Action *Action `yaml:"action"` -} - -type Environment struct { - Path []string `yaml:"path"` - PythonPath []string `yaml:"pythonPath"` -} - -type Action struct { - Command []string -} - -func (it *Config) RootDirectory() string { - return it.Root -} - -func (it *Config) IgnoreFiles() []string { - if it.Ignored == nil { - return []string{} - } - result := make([]string, 0, len(it.Ignored)) - for _, entry := range it.Ignored { - result = append(result, filepath.Join(it.Root, entry)) - } - return result -} - -func (it *Config) AvailableTasks() []string { - result := make([]string, 0, len(it.Activities)) - for name, _ := range it.Activities { - result = append(result, name) - } - sort.Strings(result) - return result -} - -func (it *Config) DefaultTask() Task { - if len(it.Activities) != 1 { - return nil - } - var result *Activity - for _, value := range it.Activities { - result = value - break - } - return result -} - -func (it *Config) TaskByName(key string) Task { - if len(key) == 0 { - return it.DefaultTask() - } - found, ok := it.Activities[key] - if ok { - return found - } - caseless := strings.ToLower(key) - for name, value := range it.Activities { - if caseless == strings.ToLower(name) { - return value - } - } - return nil -} - -func (it *Config) UsesConda() bool { - return len(it.Conda) > 0 -} - -func (it *Config) CondaConfigFile() string { - return filepath.Join(it.Root, it.Conda) -} - -func (it *Config) WorkingDirectory(taskname string) string { - activity := it.TaskByName(taskname) - if activity == nil { - return "" - } - return activity.WorkingDirectory(it) -} - -func (it *Config) ArtifactDirectory(taskname string) string { - activity := it.TaskByName(taskname) - if activity == nil { - return "" - } - return activity.ArtifactDirectory(it) -} - -func (it *Config) Paths(taskname string) pathlib.PathParts { - activity := it.TaskByName(taskname) - if activity == nil { - return pathlib.PathFrom() - } - return activity.Paths(it) -} - -func (it *Config) PythonPaths(taskname string) pathlib.PathParts { - activity := it.TaskByName(taskname) - if activity == nil { - return pathlib.PathFrom() - } - return activity.PythonPaths(it) -} - -func (it *Config) SearchPath(taskname, location string) pathlib.PathParts { - activity := it.TaskByName(taskname) - if activity == nil { - return pathlib.PathFrom() - } - return activity.SearchPath(it, location) -} - -func (it *Config) ExecutionEnvironment(taskname, location string, inject []string, full bool) []string { - activity := it.TaskByName(taskname) - if activity == nil { - return []string{} - } - return activity.ExecutionEnvironment(it, location, inject, full) -} - -func (it *Config) Validate() (bool, error) { - if it.Activities == nil { - return false, errors.New("In package.yaml, 'activities:' is required!") - } - if len(it.Activities) == 0 { - return false, errors.New("In package.yaml, 'activities:' must have at least one activity defined!") - } - if it.Conda == "" { - return false, errors.New("In package.yaml, 'condaConfig:' is required!") - } - for name, activity := range it.Activities { - if activity.Output == "" { - message := fmt.Sprintf("In package.yaml, 'output:' is required for activity %s!", name) - return false, errors.New(message) - } - if activity.Root == "" { - message := fmt.Sprintf("In package.yaml, 'activityRoot:' is required for activity %s!", name) - return false, errors.New(message) - } - if activity.Action == nil { - message := fmt.Sprintf("In package.yaml, 'action:' is required for activity %s!", name) - return false, errors.New(message) - } - if activity.Action.Command == nil { - message := fmt.Sprintf("In package.yaml, 'action/command:' is required for activity %s!", name) - return false, errors.New(message) - } - if len(activity.Action.Command) == 0 { - message := fmt.Sprintf("In package.yaml, 'action/command:' cannot be empty for activity %s!", name) - return false, errors.New(message) - } - } - return true, nil -} - -func (it *Activity) Commandline() []string { - return it.Action.Command -} - -func (it *Activity) WorkingDirectory(base Robot) string { - return filepath.Join(base.RootDirectory(), it.Root) -} - -func (it *Activity) ArtifactDirectory(base Robot) string { - return filepath.Join(it.WorkingDirectory(base), it.Output) -} - -func (it *Activity) Paths(base Robot) pathlib.PathParts { - if it.Environment == nil { - return pathlib.PathFrom() - } - return pathBuilder(base.RootDirectory(), it.Environment.Path) -} - -func (it *Activity) PythonPaths(base Robot) pathlib.PathParts { - if it.Environment == nil { - return pathlib.PathFrom() - } - return pathBuilder(base.RootDirectory(), it.Environment.PythonPath) -} - -func (it *Activity) SearchPath(base Robot, location string) pathlib.PathParts { - return conda.FindPath(location).Prepend(it.Paths(base)...) -} - -func PlainEnvironment(inject []string, full bool) []string { - environment := make([]string, 0, 100) - if full { - environment = append(environment, os.Environ()...) - } - environment = append(environment, inject...) - return environment -} - -func (it *Activity) ExecutionEnvironment(base Robot, location string, inject []string, full bool) []string { - pythonPath := it.PythonPaths(base) - environment := PlainEnvironment(inject, full) - searchPath := it.SearchPath(base, location) - python, ok := searchPath.Which("python3", conda.FileExtensions) - if !ok { - python, ok = searchPath.Which("python", conda.FileExtensions) - } - if ok { - environment = append(environment, "PYTHON_EXE="+python) - } - return append(environment, - "CONDA_DEFAULT_ENV=rcc", - "CONDA_EXE="+conda.BinConda(), - "CONDA_PREFIX="+location, - "CONDA_PROMPT_MODIFIER=(rcc)", - "CONDA_PYTHON_EXE="+conda.BinPython(), - "CONDA_SHLVL=1", - "PYTHONHOME=", - "PYTHONSTARTUP=", - "PYTHONEXECUTABLE=", - "PYTHONNOUSERSITE=1", - "ROBOCORP_HOME="+conda.RobocorpHome(), - searchPath.AsEnvironmental("PATH"), - pythonPath.AsEnvironmental("PYTHONPATH"), - fmt.Sprintf("ROBOT_ROOT=%s", it.WorkingDirectory(base)), - fmt.Sprintf("ROBOT_ARTIFACTS=%s", it.ArtifactDirectory(base)), - ) -} - -func ActivityPackageFrom(content []byte) (*Config, error) { - config := Config{} - err := yaml.Unmarshal(content, &config) - if err != nil { - return nil, err - } - return &config, nil -} - -func LoadActivityPackage(filename string) (Robot, error) { - fullpath, err := filepath.Abs(filename) - if err != nil { - return nil, err - } - content, err := ioutil.ReadFile(fullpath) - if err != nil { - return nil, err - } - config, err := ActivityPackageFrom(content) - if err != nil { - return nil, err - } - config.Root = filepath.Dir(fullpath) - return config, nil -} diff --git a/robot/config_test.go b/robot/config_test.go deleted file mode 100644 index 9fecbd6d..00000000 --- a/robot/config_test.go +++ /dev/null @@ -1,134 +0,0 @@ -package robot_test - -import ( - "strings" - "testing" - - "github.com/robocorp/rcc/hamlet" - "github.com/robocorp/rcc/robot" -) - -const ( - minimalActivity = ` -activities: - Main activity: - output: output - activityRoot: .` -) - -func TestCannotReadMissingActivityConfig(t *testing.T) { - must_be, wont_be := hamlet.Specifications(t) - - sut, err := robot.LoadYamlConfiguration("testdata/badmissing.yaml") - wont_be.Nil(err) - must_be.Nil(sut) - - sut, err = robot.LoadActivityPackage("testdata/bad.yaml") - wont_be.Nil(err) - must_be.Nil(sut) -} - -func TestCanReadTemplateActivityConfig(t *testing.T) { - must_be, wont_be := hamlet.Specifications(t) - - raw, err := robot.LoadActivityPackage("testdata/template.yaml") - must_be.Nil(err) - wont_be.Nil(raw) - sut := raw.(*robot.Config) - must_be.Equal("config/conda.yaml", sut.Conda) - must_be.True(strings.HasSuffix(sut.CondaConfigFile(), "config/conda.yaml")) - must_be.Equal(1, len(sut.Activities)) - activity := sut.DefaultTask().(*robot.Activity) - wont_be.Nil(activity) - must_be.True(strings.HasSuffix(activity.WorkingDirectory(sut), "/testdata")) - must_be.Nil(sut.TaskByName("Missing Activity Name")) - must_be.Same(activity, sut.TaskByName("My activity")) - must_be.Same(activity, sut.TaskByName("my Activity")) - must_be.Equal([]string{"My activity"}, sut.AvailableTasks()) - must_be.True(strings.HasSuffix(activity.ArtifactDirectory(sut), "output")) - must_be.Equal(".", activity.Root) - wont_be.Nil(activity.Environment) - wont_be.Nil(activity.Action) - must_be.Equal(10, len(activity.Commandline())) - must_be.Equal(1, len(activity.Paths(sut))) - must_be.True(len(activity.ExecutionEnvironment(sut, "foobar", []string{}, false)) > 3) - must_be.True(len(activity.ExecutionEnvironment(sut, "foobar", []string{}, true)) > len(activity.ExecutionEnvironment(sut, "foobar", []string{}, false))) - must_be.Equal(3, len(activity.PythonPaths(sut))) - ok, err := sut.Validate() - must_be.True(ok) - must_be.Nil(err) -} - -func TestCanReadComplexActivityConfig(t *testing.T) { - must_be, wont_be := hamlet.Specifications(t) - - raw, err := robot.LoadActivityPackage("testdata/complex.yaml") - must_be.Nil(err) - wont_be.Nil(raw) - sut := raw.(*robot.Config) - wont_be.Nil(sut.IgnoreFiles()) - must_be.Equal(2, len(sut.IgnoreFiles())) - must_be.Nil(sut.DefaultTask()) - must_be.Equal("conda.yaml", sut.Conda) - must_be.Equal(2, len(sut.Activities)) - wont_be.Nil(sut.TaskByName("Read Excel to work item")) - wont_be.Nil(sut.TaskByName("Generate PDFs from work item")) - must_be.Equal([]string{"Generate PDFs from work item", "Read Excel to work item"}, sut.AvailableTasks()) - ok, err := sut.Validate() - must_be.True(ok) - must_be.Nil(err) -} - -func TestCanReadEnvironmentlessActivityConfig(t *testing.T) { - must_be, wont_be := hamlet.Specifications(t) - - raw, err := robot.LoadActivityPackage("testdata/complex.yaml") - must_be.Nil(err) - wont_be.Nil(raw) - sut := raw.(*robot.Config) - activity := sut.DefaultTask() - must_be.Nil(activity) - ok, err := sut.Validate() - must_be.True(ok) - must_be.Nil(err) -} - -func TestCanReadActivityConfigFromText(t *testing.T) { - must_be, wont_be := hamlet.Specifications(t) - - sut, err := robot.ActivityPackageFrom([]byte("")) - must_be.Nil(err) - wont_be.Nil(sut) - must_be.Nil(sut.DefaultTask()) - must_be.Nil(sut.TaskByName("")) - must_be.Nil(sut.TaskByName("foo")) - must_be.Equal("", sut.CondaConfigFile()) - must_be.Equal("", sut.CondaConfigFile()) - ok, err := sut.Validate() - wont_be.True(ok) - wont_be.Nil(err) -} - -func TestCanParseMinimalActivityConfig(t *testing.T) { - must_be, wont_be := hamlet.Specifications(t) - - sut, err := robot.ActivityPackageFrom([]byte(minimalActivity)) - must_be.Nil(err) - wont_be.Nil(sut) - wont_be.Nil(sut.DefaultTask()) - wont_be.Nil(sut.TaskByName("")) - must_be.Nil(sut.TaskByName("foo")) - must_be.Equal("", sut.CondaConfigFile()) - must_be.Equal("", sut.CondaConfigFile()) - ok, err := sut.Validate() - wont_be.True(ok) - wont_be.Nil(err) -} - -func TestCanReadActivityConfigFromBadText(t *testing.T) { - must_be, wont_be := hamlet.Specifications(t) - - sut, err := robot.ActivityPackageFrom([]byte(":")) - wont_be.Nil(err) - must_be.Nil(sut) -} diff --git a/robot/robot.go b/robot/robot.go index 7cca32a1..92b5550a 100644 --- a/robot/robot.go +++ b/robot/robot.go @@ -3,9 +3,10 @@ package robot import ( "errors" "fmt" - "io/ioutil" "os" "path/filepath" + "regexp" + "runtime" "sort" "strings" @@ -13,11 +14,16 @@ import ( "github.com/robocorp/rcc/conda" "github.com/robocorp/rcc/pathlib" "github.com/robocorp/rcc/pretty" + "github.com/robocorp/rcc/shell" - "github.com/google/shlex" "gopkg.in/yaml.v2" ) +var ( + GoosPattern = regexp.MustCompile("(?i:(windows|darwin|linux))") + GoarchPattern = regexp.MustCompile("(?i:(amd64|arm64))") +) + type Robot interface { IgnoreFiles() []string AvailableTasks() []string @@ -25,42 +31,208 @@ type Robot interface { TaskByName(string) Task UsesConda() bool CondaConfigFile() string + PreRunScripts() []string RootDirectory() string + HasHolozip() bool + Holozip() string Validate() (bool, error) + Diagnostics(*common.DiagnosticStatus, bool) + DependenciesFile() (string, bool) - // compatibility "string" argument (task name) - WorkingDirectory(string) string - ArtifactDirectory(string) string - Paths(string) pathlib.PathParts - PythonPaths(string) pathlib.PathParts - SearchPath(taskname, location string) pathlib.PathParts - ExecutionEnvironment(taskname, location string, inject []string, full bool) []string + WorkingDirectory() string + ArtifactDirectory() string + FreezeFilename() string + Paths() pathlib.PathParts + PythonPaths() pathlib.PathParts + SearchPath(location string) pathlib.PathParts + RobotExecutionEnvironment(location string, inject []string, full bool) []string } type Task interface { - WorkingDirectory(Robot) string - ArtifactDirectory(Robot) string - Paths(Robot) pathlib.PathParts - PythonPaths(Robot) pathlib.PathParts - SearchPath(Robot, string) pathlib.PathParts - ExecutionEnvironment(robot Robot, location string, inject []string, full bool) []string Commandline() []string } type robot struct { - Tasks map[string]*task `yaml:"tasks"` - Conda string `yaml:"condaConfigFile"` - Ignored []string `yaml:"ignoreFiles"` - Artifacts string `yaml:"artifactsDir"` - Path []string `yaml:"PATH"` - Pythonpath []string `yaml:"PYTHONPATH"` - Root string + Tasks map[string]*task `yaml:"tasks"` + Devtasks map[string]*task `yaml:"devTasks"` + Conda string `yaml:"condaConfigFile,omitempty"` + PreRun []string `yaml:"preRunScripts,omitempty"` + Environments []string `yaml:"environmentConfigs,omitempty"` + Ignored []string `yaml:"ignoreFiles"` + Artifacts string `yaml:"artifactsDir"` + Path []string `yaml:"PATH"` + Pythonpath []string `yaml:"PYTHONPATH"` + Root string } type task struct { Task string `yaml:"robotTaskName,omitempty"` Shell string `yaml:"shell,omitempty"` Command []string `yaml:"command,omitempty"` + robot *robot +} + +func (it *robot) taskMap(note bool) map[string]*task { + if common.DeveloperFlag { + if note { + pretty.Note("Operating in developer mode. Using 'devTasks:' instead of 'tasks:'.") + } + return it.Devtasks + } else { + return it.Tasks + } +} + +func (it *robot) relink() { + for _, task := range it.Tasks { + if task != nil { + task.robot = it + } + } + for _, task := range it.Devtasks { + if task != nil { + task.robot = it + } + } +} + +func (it *robot) diagnoseTasks(diagnose common.Diagnoser) { + if it.Tasks == nil { + diagnose.Fail(0, "", "Missing 'tasks:' from robot.yaml.") + return + } + ok := true + if len(it.Tasks) == 0 { + diagnose.Fail(0, "", "There must be at least one task defined in 'tasks:' section in robot.yaml.") + ok = false + } else { + diagnose.Ok(0, "Tasks are defined in robot.yaml") + } + for name, task := range it.Tasks { + count := 0 + if len(task.Task) > 0 { + count += 1 + } + if len(task.Shell) > 0 { + count += 1 + } + if task.Command != nil && len(task.Command) > 0 { + count += 1 + } + if count != 1 { + diagnose.Fail(0, "", "In robot.yaml, task '%s' needs exactly one of robotTaskName/shell/command definition!", name) + ok = false + } + } + if ok { + diagnose.Ok(0, "Each task has exactly one definition.") + } +} + +func (it *robot) diagnoseVariousPaths(diagnose common.Diagnoser) { + ok := true + for _, path := range it.Path { + if filepath.IsAbs(path) { + diagnose.Fail(0, "", "PATH entry %q seems to be absolute, which makes robot machine dependent.", path) + ok = false + } + } + if ok { + diagnose.Ok(0, "PATH settings in robot.yaml are ok.") + } + ok = true + for _, path := range it.Pythonpath { + if filepath.IsAbs(path) { + diagnose.Fail(0, "", "PYTHONPATH entry %q seems to be absolute, which makes robot machine dependent.", path) + ok = false + } + } + if ok { + diagnose.Ok(0, "PYTHONPATH settings in robot.yaml are ok.") + } + ok = true + if it.Ignored == nil || len(it.Ignored) == 0 { + diagnose.Warning(0, "", "No ignoreFiles defined, so everything ends up inside robot.zip file.") + ok = false + } else { + for at, path := range it.Ignored { + if len(strings.TrimSpace(path)) == 0 { + diagnose.Fail(0, "", "there is empty entry in ignoreFiles at position %d", at+1) + ok = false + continue + } + if filepath.IsAbs(path) { + diagnose.Fail(0, "", "ignoreFiles entry %q seems to be absolute, which makes robot machine dependent.", path) + ok = false + } + } + for _, path := range it.IgnoreFiles() { + _, err := pathlib.LoadIgnoreFile(path, true) + if err != nil { + diagnose.Warning(0, "", "Could not load ignoreFiles entry %q, reason: %v", filepath.Base(path), err) + ok = false + continue + } + } + for _, path := range it.IgnoreFiles() { + if !pathlib.IsFile(path) { + diagnose.Fail(0, "", "ignoreFiles entry %q is not a file.", path) + ok = false + } + } + } + if ok { + diagnose.Ok(0, "ignoreFiles settings in robot.yaml are ok.") + } +} + +func (it *robot) Diagnostics(target *common.DiagnosticStatus, production bool) { + diagnose := target.Diagnose("Robot") + it.diagnoseTasks(diagnose) + it.diagnoseVariousPaths(diagnose) + inside, err := common.IsInsideProductHome(it.WorkingDirectory()) + if err == nil && inside { + diagnose.Warning(0, "", "Robot working directory %q is inside %s (%s)", it.WorkingDirectory(), common.Product.HomeVariable(), common.Product.Home()) + } + if it.Artifacts == "" { + diagnose.Fail(0, "", "In robot.yaml, 'artifactsDir:' is required!") + } else { + if filepath.IsAbs(it.Artifacts) { + diagnose.Fail(0, "", "artifactDir %q seems to be absolute, which makes robot machine dependent.", it.Artifacts) + } else { + diagnose.Ok(0, "Artifacts directory defined in robot.yaml") + } + } + effectiveConfig := it.CondaConfigFile() + if effectiveConfig == "" { + diagnose.Ok(0, "In robot.yaml, effective environment configuration is missing. So this is shell robot.") + target.Details["cacheable-environment-configuration"] = "false" + } else { + diagnose.Ok(0, "In robot.yaml, environment configuration %q is present. So this is python robot.", effectiveConfig) + condaEnv, err := conda.ReadPackageCondaYaml(effectiveConfig) + if err != nil { + diagnose.Fail(0, "", "From robot.yaml, loading conda.yaml failed with: %v", err) + } else { + condaEnv.Diagnostics(target, production) + } + } + target.Details["robot-use-conda"] = fmt.Sprintf("%v", it.UsesConda()) + target.Details["robot-conda-file"] = it.CondaConfigFile() + target.Details["hololib.zip"] = it.Holozip() + target.Details["robot-root-directory"] = it.RootDirectory() + target.Details["robot-working-directory"] = it.WorkingDirectory() + target.Details["robot-artifact-directory"] = it.ArtifactDirectory() + target.Details["robot-paths"] = strings.Join(it.Paths(), ", ") + target.Details["robot-python-paths"] = strings.Join(it.PythonPaths(), ", ") + dependencies, ok := it.DependenciesFile() + if !ok { + dependencies = "missing" + } else { + if it.VerifyCondaDependencies() { + diagnose.Ok(0, "Dependencies in conda.yaml and dependencies.yaml match.") + } + } + target.Details["robot-dependencies-yaml"] = dependencies } func (it *robot) Validate() (bool, error) { @@ -85,42 +257,92 @@ func (it *robot) Validate() (bool, error) { count += 1 } if count != 1 { - return false, errors.New(fmt.Sprintf("In robot.yaml, task '%s' needs exactly one of robotTaskName/shell/command definition!", name)) + return false, fmt.Errorf("In robot.yaml, task '%s' needs exactly one of robotTaskName/shell/command definition!", name) } } return true, nil } +func (it *robot) DependenciesFile() (string, bool) { + filename := filepath.Join(it.Root, "dependencies.yaml") + return filename, pathlib.IsFile(filename) +} + +func (it *robot) VerifyCondaDependencies() bool { + wanted, ok := it.DependenciesFile() + if !ok { + return true + } + dependencies := conda.LoadWantedDependencies(wanted) + if len(dependencies) == 0 { + return true + } + condaEnv, err := conda.ReadPackageCondaYaml(it.CondaConfigFile()) + if err != nil { + return true + } + ideal, ok := condaEnv.FromDependencies(dependencies) + if !ok { + body, err := ideal.AsYaml() + if err == nil { + fmt.Println("IDEAL:", body) + } + } + return ok +} + func (it *robot) RootDirectory() string { return it.Root } +func (it *robot) HasHolozip() bool { + return len(it.Holozip()) > 0 +} + +func (it *robot) Holozip() string { + zippath := filepath.Join(it.Root, "hololib.zip") + if pathlib.IsFile(zippath) { + return zippath + } + return "" +} + func (it *robot) IgnoreFiles() []string { if it.Ignored == nil { return []string{} } result := make([]string, 0, len(it.Ignored)) - for _, entry := range it.Ignored { - result = append(result, filepath.Join(it.Root, entry)) + for at, entry := range it.Ignored { + if len(strings.TrimSpace(entry)) == 0 { + pretty.Warning("Ignore file entry at position %d is empty string!", at+1) + continue + } + fullpath := filepath.Join(it.Root, entry) + if !pathlib.IsFile(fullpath) { + pretty.Warning("Ignore file %q is not a file!", fullpath) + } + result = append(result, fullpath) } return result } func (it *robot) AvailableTasks() []string { - result := make([]string, 0, len(it.Tasks)) - for name, _ := range it.Tasks { - result = append(result, name) + tasks := it.taskMap(false) + result := make([]string, 0, len(tasks)) + for name, _ := range tasks { + result = append(result, fmt.Sprintf("%q", name)) } sort.Strings(result) return result } func (it *robot) DefaultTask() Task { - if len(it.Tasks) != 1 { + tasks := it.taskMap(true) + if len(tasks) != 1 { return nil } var result *task - for _, value := range it.Tasks { + for _, value := range tasks { result = value break } @@ -131,13 +353,14 @@ func (it *robot) TaskByName(name string) Task { if len(name) == 0 { return it.DefaultTask() } - key := strings.TrimSpace(name) - found, ok := it.Tasks[key] + tasks := it.taskMap(true) + key := strings.Trim(name, "\t\r\n\"' ") + found, ok := tasks[key] if ok { return found } caseless := strings.ToLower(key) - for name, value := range it.Tasks { + for name, value := range tasks { if caseless == strings.ToLower(strings.TrimSpace(name)) { return value } @@ -146,18 +369,75 @@ func (it *robot) TaskByName(name string) Task { } func (it *robot) UsesConda() bool { - return len(it.Conda) > 0 + return len(it.Conda) > 0 || len(it.availableEnvironmentConfigurations(common.Platform())) > 0 } func (it *robot) CondaConfigFile() string { + resolved := it.resolveCondaConfigFile() + pretty.JustOnce("Note! For now, resolved effective environment configuration file is %q.", resolved) + return resolved +} + +func (it *robot) resolveCondaConfigFile() string { + available := it.availableEnvironmentConfigurations(common.Platform()) + if len(available) > 0 { + return available[0] + } return filepath.Join(it.Root, it.Conda) } -func (it *robot) WorkingDirectory(string) string { +func (it *robot) PreRunScripts() []string { + return it.PreRun +} + +func (it *robot) WorkingDirectory() string { return it.Root } -func (it *robot) ArtifactDirectory(string) string { +func freezeFileBasename() string { + return fmt.Sprintf("environment_%s_freeze.yaml", common.Platform()) +} + +func submatch(pattern *regexp.Regexp, expected, text string) bool { + match := pattern.FindStringSubmatch(text) + return match == nil || len(match) == 0 || match[0] == expected +} + +func PlatformAcceptableFile(architecture, operatingSystem, filename string) bool { + return submatch(GoarchPattern, architecture, filename) && submatch(GoosPattern, operatingSystem, filename) +} + +func (it *robot) availableEnvironmentConfigurations(marker string) []string { + result := make([]string, 0, len(it.Environments)) + common.Trace("Available environment configurations:") + for _, part := range it.Environments { + underscored := strings.Count(part, "_") > 2 + freezed := strings.Contains(strings.ToLower(part), "freeze") + marked := strings.Contains(part, marker) + if (underscored || freezed) && !marked { + continue + } + if !PlatformAcceptableFile(runtime.GOARCH, runtime.GOOS, part) { + continue + } + fullpath := filepath.Join(it.Root, part) + if !pathlib.IsFile(fullpath) { + continue + } + common.Trace("- %s", fullpath) + result = append(result, fullpath) + } + if len(result) == 0 { + common.Trace("- nothing") + } + return result +} + +func (it *robot) FreezeFilename() string { + return filepath.Join(it.ArtifactDirectory(), freezeFileBasename()) +} + +func (it *robot) ArtifactDirectory() string { return filepath.Join(it.Root, it.Artifacts) } @@ -177,83 +457,36 @@ func pathBuilder(root string, tails []string) pathlib.PathParts { return pathlib.PathFrom(result...) } -func (it *robot) Paths(string) pathlib.PathParts { +func (it *robot) Paths() pathlib.PathParts { if it == nil { return pathlib.PathFrom() } return pathBuilder(it.Root, it.Path) } -func (it *robot) PythonPaths(string) pathlib.PathParts { +func (it *robot) PythonPaths() pathlib.PathParts { if it == nil { return pathlib.PathFrom() } return pathBuilder(it.Root, it.Pythonpath) } -func (it *robot) SearchPath(taskname, location string) pathlib.PathParts { - return conda.FindPath(location).Prepend(it.Paths("")...) +func (it *robot) SearchPath(location string) pathlib.PathParts { + return conda.FindPath(location).Prepend(it.Paths()...) } -func (it *robot) ExecutionEnvironment(taskname, location string, inject []string, full bool) []string { - environment := make([]string, 0, 100) - if full { - environment = append(environment, os.Environ()...) - } - environment = append(environment, inject...) - searchPath := it.SearchPath(taskname, location) - python, ok := searchPath.Which("python3", conda.FileExtensions) - if !ok { - python, ok = searchPath.Which("python", conda.FileExtensions) - } - if ok { - environment = append(environment, "PYTHON_EXE="+python) - } +func (it *robot) RobotExecutionEnvironment(location string, inject []string, full bool) []string { + environment := conda.CondaExecutionEnvironment(location, inject, full) return append(environment, - "CONDA_DEFAULT_ENV=rcc", - "CONDA_EXE="+conda.BinConda(), - "CONDA_PREFIX="+location, - "CONDA_PROMPT_MODIFIER=(rcc)", - "CONDA_PYTHON_EXE="+conda.BinPython(), - "CONDA_SHLVL=1", - "PYTHONHOME=", - "PYTHONSTARTUP=", - "PYTHONEXECUTABLE=", - "PYTHONNOUSERSITE=1", - "ROBOCORP_HOME="+conda.RobocorpHome(), - searchPath.AsEnvironmental("PATH"), - it.PythonPaths("").AsEnvironmental("PYTHONPATH"), - fmt.Sprintf("ROBOT_ROOT=%s", it.WorkingDirectory("")), - fmt.Sprintf("ROBOT_ARTIFACTS=%s", it.ArtifactDirectory("")), + it.SearchPath(location).AsEnvironmental("PATH"), + it.PythonPaths().AsEnvironmental("PYTHONPATH"), + fmt.Sprintf("ROBOT_ROOT=%s", it.WorkingDirectory()), + fmt.Sprintf("ROBOT_ARTIFACTS=%s", it.ArtifactDirectory()), ) } -func (it *task) WorkingDirectory(robot Robot) string { - return robot.WorkingDirectory("") -} - -func (it *task) ArtifactDirectory(robot Robot) string { - return robot.ArtifactDirectory("") -} - -func (it *task) SearchPath(robot Robot, location string) pathlib.PathParts { - return robot.SearchPath("", location) -} - -func (it *task) Paths(robot Robot) pathlib.PathParts { - return robot.Paths("") -} - -func (it *task) PythonPaths(robot Robot) pathlib.PathParts { - return robot.PythonPaths("") -} - -func (it *task) ExecutionEnvironment(robot Robot, location string, inject []string, full bool) []string { - return robot.ExecutionEnvironment("", location, inject, full) -} - func (it *task) shellCommand() []string { - result, err := shlex.Split(it.Shell) + result, err := shell.Split(it.Shell) if err != nil { common.Log("Shell parsing failure: %v with command %v", err, it.Shell) return []string{} @@ -262,11 +495,15 @@ func (it *task) shellCommand() []string { } func (it *task) taskCommand() []string { + output := "output" + if it.robot != nil { + output = it.robot.Artifacts + } return []string{ "python", "-m", "robot", "--report", "NONE", - "--outputdir", "output", + "--outputdir", output, "--logtitle", "Task log", "--task", it.Task, ".", @@ -289,42 +526,43 @@ func robotFrom(content []byte) (*robot, error) { if err != nil { return nil, err } + config.relink() return &config, nil } -func LoadRobotYaml(filename string) (Robot, error) { +func PlainEnvironment(inject []string, full bool) []string { + environment := make([]string, 0, 100) + if full { + environment = append(environment, os.Environ()...) + } + environment = append(environment, inject...) + return environment +} + +func LoadRobotYaml(filename string, visible bool) (Robot, error) { fullpath, err := filepath.Abs(filename) if err != nil { - return nil, err + return nil, fmt.Errorf("%q: %w", filename, err) } - content, err := ioutil.ReadFile(fullpath) + content, err := os.ReadFile(fullpath) if err != nil { - return nil, err + return nil, fmt.Errorf("%q: %w", fullpath, err) + } + if visible { + common.Log("%q as robot.yaml is:\n%s", fullpath, string(content)) } robot, err := robotFrom(content) if err != nil { - return nil, err + return nil, fmt.Errorf("%q: %w", fullpath, err) } robot.Root = filepath.Dir(fullpath) return robot, nil } -func LoadYamlConfiguration(filename string) (Robot, error) { - if strings.HasSuffix(filename, "package.yaml") { - common.Log("%sWARNING! Support for 'package.yaml' is deprecated. Upgrade to 'robot.yaml'!%s", pretty.Red, pretty.Reset) - return LoadActivityPackage(filename) - } - return LoadRobotYaml(filename) -} - func DetectConfigurationName(directory string) string { robot, err := pathlib.FindNamedPath(directory, "robot.yaml") if err == nil && len(robot) > 0 { return robot } - robot, err = pathlib.FindNamedPath(directory, "package.yaml") - if err == nil && len(robot) > 0 { - return robot - } - return filepath.Join(directory, "package.yaml") + return filepath.Join(directory, "robot.yaml") } diff --git a/robot/robot_test.go b/robot/robot_test.go index 3056abd3..fa0afc41 100644 --- a/robot/robot_test.go +++ b/robot/robot_test.go @@ -11,15 +11,61 @@ import ( func TestCannotReadMissingRobotYaml(t *testing.T) { must, wont := hamlet.Specifications(t) - sut, err := robot.LoadRobotYaml("testdata/badmissing.yaml") + sut, err := robot.LoadRobotYaml("testdata/badmissing.yaml", false) wont.Nil(err) must.Nil(sut) } +func TestCanAcceptPlatformFiles(t *testing.T) { + must, wont := hamlet.Specifications(t) + + for _, filename := range []string{"anyscript", "anyscript.bat", "anyscript.cmd"} { + must.True(robot.PlatformAcceptableFile("amd64", "linux", filename)) + must.True(robot.PlatformAcceptableFile("amd64", "windows", filename)) + must.True(robot.PlatformAcceptableFile("amd64", "darwin", filename)) + must.True(robot.PlatformAcceptableFile("arm64", "linux", filename)) + must.True(robot.PlatformAcceptableFile("arm64", "windows", filename)) + must.True(robot.PlatformAcceptableFile("arm64", "darwin", filename)) + } + + for _, filename := range []string{"any.bat", "any.cmd", "any.sh", "at_linux.sh", "at_arm64.sh", "at_arm64_linux.sh"} { + must.True(robot.PlatformAcceptableFile("arm64", "linux", filename)) + } + + for _, filename := range []string{"any.bat", "any.cmd", "any.sh", "at_darwin.sh", "at_arm64.sh", "at_arm64_darwin.sh"} { + must.True(robot.PlatformAcceptableFile("arm64", "darwin", filename)) + } + + for _, filename := range []string{"any.bat", "any.cmd", "any.sh", "at_windows.sh", "at_amd64.sh", "at_amd64_windows.sh"} { + must.True(robot.PlatformAcceptableFile("amd64", "windows", filename)) + } + + for _, filename := range []string{"at_arm64.sh", "at_windows.bat", "at_darwin.sh", "at_arm64_linux.sh"} { + wont.True(robot.PlatformAcceptableFile("amd64", "linux", filename)) + } + + for _, filename := range []string{"at_linux.sh", "at_arm64.sh", "at_amd64_darwin.sh", "at_amd64_linux.sh", "at_arm64_windows.sh"} { + wont.True(robot.PlatformAcceptableFile("amd64", "windows", filename)) + } + + for _, filename := range []string{"at_linux.sh", "at_arm64.sh", "at_arm64_darwin.sh", "at_amd64_linux.sh", "at_amd64_windows.sh"} { + wont.True(robot.PlatformAcceptableFile("amd64", "darwin", filename)) + } +} + +func TestCanMatchArchitecture(t *testing.T) { + must, wont := hamlet.Specifications(t) + + wont.Nil(robot.GoosPattern) + wont.Nil(robot.GoarchPattern) + must.Equal([]string{"darwin", "darwin"}, robot.GoosPattern.FindStringSubmatch("foo_darwin_arm64_freeze.yaml")) + must.Equal([]string{"arm64", "arm64"}, robot.GoarchPattern.FindStringSubmatch("foo_darwin_arm64_freeze.yaml")) +} + func TestCanReadRealRobotYaml(t *testing.T) { must, wont := hamlet.Specifications(t) - sut, err := robot.LoadRobotYaml("testdata/robot.yaml") + sut, err := robot.LoadRobotYaml("testdata/robot.yaml", false) must.Nil(err) wont.Nil(sut) must.Equal(1, len(sut.IgnoreFiles())) @@ -30,12 +76,12 @@ func TestCanReadRealRobotYaml(t *testing.T) { wont.Nil(sut.TaskByName("task form name")) wont.Nil(sut.TaskByName("Shell Form Name")) wont.Nil(sut.TaskByName(" Old command form name ")) - must.Equal(1, len(sut.Paths(""))) - must.Equal(3, len(sut.PythonPaths(""))) - must.True(2 < len(sut.SearchPath("", "."))) + must.Equal(1, len(sut.Paths())) + must.Equal(3, len(sut.PythonPaths())) + must.True(2 < len(sut.SearchPath("."))) must.True(strings.HasSuffix(sut.CondaConfigFile(), "conda.yaml")) - must.True(strings.HasSuffix(sut.WorkingDirectory(""), "testdata")) - must.True(strings.HasSuffix(sut.ArtifactDirectory(""), "output")) + must.True(strings.HasSuffix(sut.WorkingDirectory(), "testdata")) + must.True(strings.HasSuffix(sut.ArtifactDirectory(), "output")) valid, err := sut.Validate() must.True(valid) must.Nil(err) @@ -44,7 +90,7 @@ func TestCanReadRealRobotYaml(t *testing.T) { func TestCanGetShellFormCommand(t *testing.T) { must, wont := hamlet.Specifications(t) - sut, err := robot.LoadRobotYaml("testdata/robot.yaml") + sut, err := robot.LoadRobotYaml("testdata/robot.yaml", false) must.Nil(err) wont.Nil(sut) task := sut.TaskByName("Shell Form Name") @@ -59,7 +105,7 @@ func TestCanGetShellFormCommand(t *testing.T) { func TestCanGetTaskFormCommand(t *testing.T) { must, wont := hamlet.Specifications(t) - sut, err := robot.LoadRobotYaml("testdata/robot.yaml") + sut, err := robot.LoadRobotYaml("testdata/robot.yaml", false) must.Nil(err) wont.Nil(sut) task := sut.TaskByName("Task Form Name") diff --git a/robot/setup.go b/robot/setup.go index 727e5953..f86d9094 100644 --- a/robot/setup.go +++ b/robot/setup.go @@ -2,7 +2,7 @@ package robot import ( "fmt" - "io/ioutil" + "os" "path/filepath" "gopkg.in/yaml.v2" @@ -36,11 +36,11 @@ func LoadEnvironmentSetup(filename string) (Setup, error) { } fullpath, err := filepath.Abs(filename) if err != nil { - return nil, err + return nil, fmt.Errorf("%q: %w", filename, err) } - content, err := ioutil.ReadFile(fullpath) + content, err := os.ReadFile(fullpath) if err != nil { - return nil, err + return nil, fmt.Errorf("%q: %w", fullpath, err) } return EnvironmentSetupFrom(content) } diff --git a/robot_requirements.txt b/robot_requirements.txt index 1f683238..2982349f 100644 --- a/robot_requirements.txt +++ b/robot_requirements.txt @@ -1 +1 @@ -robotframework==3.1.2 +robotframework==6.1.1 diff --git a/robot_tests/bare_action/package.yaml b/robot_tests/bare_action/package.yaml new file mode 100644 index 00000000..2f011f63 --- /dev/null +++ b/robot_tests/bare_action/package.yaml @@ -0,0 +1,14 @@ +name: RCC testing package +description: Just for testing rcc. +version: 0.0.1 + +documentation: https://github.com/robocorp/rcc/ + +dependencies: + conda-forge: + - python=3.10.12 + - uv=0.2.5 + pypi: + - robocorp=1.6.2 + - robocorp-actions=0.0.7 + diff --git a/robot_tests/bug_reports.robot b/robot_tests/bug_reports.robot index 4ca5ea68..65bfaaae 100644 --- a/robot_tests/bug_reports.robot +++ b/robot_tests/bug_reports.robot @@ -7,10 +7,37 @@ Github issue 7 about initial call with do-not-track [Setup] Remove config tmp/bug_7.yaml Wont Exist tmp/bug_7.yaml - Goal First time calling rcc configure identity to disable tracking Step build/rcc configure identity --controller citests --do-not-track --config tmp/bug_7.yaml Must Have anonymous health tracking is: disabled +Bug in virtual holotree with gzipped files + Step build/rcc holotree blueprint --controller citests robot_tests/spellbug/conda.yaml + Use STDERR + Must Have Blueprint "8b2083d262262cbd" is available: false + + Step build/rcc run --liveonly --controller citests --robot robot_tests/spellbug/robot.yaml + Use STDOUT + Must Have Bug fixed! + + Step build/rcc holotree blueprint --controller citests robot_tests/spellbug/conda.yaml + Use STDERR + Must Have Blueprint "8b2083d262262cbd" is available: false + + Step build/rcc run --controller citests --robot robot_tests/spellbug/robot.yaml + Use STDOUT + Must Have Bug fixed! + + Step build/rcc holotree blueprint --controller citests robot_tests/spellbug/conda.yaml + Use STDERR + Must Have Blueprint "8b2083d262262cbd" is available: true + +Github issue 32 about rcc task script command failing + Step build/rcc task script --controller citests --robot robot_tests/spellbug/robot.yaml -- pip list + Use STDOUT + Must Have pyspellchecker + Must Have 0.6.2 + + *** Keywords *** Remove Config diff --git a/robot_tests/certificates.yaml b/robot_tests/certificates.yaml new file mode 100644 index 00000000..9c46ef58 --- /dev/null +++ b/robot_tests/certificates.yaml @@ -0,0 +1,4 @@ +channels: + - conda-forge +dependencies: + - ca-certificates diff --git a/robot_tests/conda.yaml b/robot_tests/conda.yaml new file mode 100644 index 00000000..3fa8c8e8 --- /dev/null +++ b/robot_tests/conda.yaml @@ -0,0 +1,4 @@ +channels: +- conda-forge +dependencies: +- python=3.7.8 diff --git a/robot_tests/development_process.robot b/robot_tests/development_process.robot new file mode 100644 index 00000000..0c49c60f --- /dev/null +++ b/robot_tests/development_process.robot @@ -0,0 +1,10 @@ +*** Settings *** +Resource resources.robot + +*** Test cases *** + +Has required changes in commit based on development process. + Step git show --stat + Must Have docs/changelog.md + Must Have common/version.go + diff --git a/robot_tests/documentation.robot b/robot_tests/documentation.robot new file mode 100644 index 00000000..38ba4603 --- /dev/null +++ b/robot_tests/documentation.robot @@ -0,0 +1,21 @@ +*** Settings *** +Resource resources.robot +Test template Verify documentation + +*** Test cases *** DOCUMENTATION EXPECT + +Changelog in documentation changelog troubleshooting documentation added as +Features in documentation features Incomplete list of rcc features +LICENSE in documentation license TERMS AND CONDITIONS FOR USE +Profiles in documentation profiles Profile is way to capture +Recipes in documentation recipes Tips, tricks, and recipies +Troubleshooting in documentation troubleshooting Troubleshooting guidelines and known solutions +Tutorial in documentation tutorial Welcome to RCC tutorial +Use-cases in documentation usecases Incomplete list of rcc use cases + +*** Keywords *** + +Verify documentation + [Arguments] ${document} ${expected} + Step build/rcc man ${document} --controller citests + Must Have ${expected} diff --git a/robot_tests/exitcodes.robot b/robot_tests/exitcodes.robot index 9d716396..66e4f82f 100644 --- a/robot_tests/exitcodes.robot +++ b/robot_tests/exitcodes.robot @@ -1,71 +1,46 @@ *** Settings *** -Library OperatingSystem -Test template Verify exitcodes +Library OperatingSystem +Library supporting.py -*** Test cases *** EXITCODE COMMAND +Test Template Verify exitcodes -General failure of rcc command 1 build/rcc crapiti -h --controller citests -General output for rcc command 0 build/rcc --controller citests - -Help for rcc command 0 build/rcc -h - -Help for rcc assistant subcommand 0 build/rcc assistant -h --controller citests -Help for rcc cloud subcommand 0 build/rcc cloud -h --controller citests -Help for rcc conda subcommand 0 build/rcc conda -h --controller citests -Help for rcc configure subcommand 0 build/rcc configure -h --controller citests -Help for rcc env subcommand 0 build/rcc env -h --controller citests +*** Test Cases *** EXITCODE COMMAND +General failure of rcc command 1 build/rcc crapiti -h --controller citests +General output for rcc command 0 build/rcc --controller citests +Help for rcc command 0 build/rcc -h +Help for rcc assistant subcommand 0 build/rcc assistant -h --controller citests +Help for rcc cloud subcommand 0 build/rcc cloud -h --controller citests +Help for rcc community subcommand 0 build/rcc community -h --controller citests +Help for rcc configure subcommand 0 build/rcc configure -h --controller citests +Help for rcc create subcommand 0 build/rcc create -h --controller citests Help for rcc feedback subcommand 0 build/rcc feedback -h --controller citests -Help for rcc help subcommand 0 build/rcc help -h --controller citests -Help for rcc man subcommand 0 build/rcc man -h --controller citests +Help for rcc holotree subcommand 0 build/rcc holotree -h --controller citests +Help for rcc help subcommand 0 build/rcc help -h --controller citests +Help for rcc interactive subcommand 0 build/rcc interactive -h --controller citests Help for rcc internal subcommand 0 build/rcc internal -h --controller citests -Help for rcc robot subcommand 0 build/rcc robot -h --controller citests -Help for rcc task subcommand 0 build/rcc task -h --controller citests -Help for rcc version subcommand 0 build/rcc version -h --controller citests - -Help for rcc assistant list 0 build/rcc assistant list -h --controller citests -Help for rcc assistant run 0 build/rcc assistant run -h --controller citests - -Help for rcc cloud authorize 0 build/rcc cloud authorize -h --controller citests -Help for rcc cloud download 0 build/rcc cloud download -h --controller citests -Help for rcc cloud new 0 build/rcc cloud new -h --controller citests -Help for rcc cloud pull 0 build/rcc cloud pull -h --controller citests -Help for rcc cloud push 0 build/rcc cloud push -h --controller citests -Help for rcc cloud upload 0 build/rcc cloud upload -h --controller citests -Help for rcc cloud userinfo 0 build/rcc cloud userinfo -h --controller citests -Help for rcc cloud workspace 0 build/rcc cloud workspace -h --controller citests - -Help for rcc conda check 0 build/rcc conda check -h --controller citests -Help for rcc conda download 0 build/rcc conda download -h --controller citests -Help for rcc conda install 0 build/rcc conda install -h --controller citests - -Help for rcc configure credentials 0 build/rcc configure credentials -h --controller citests +Help for rcc man subcommand 0 build/rcc man -h --controller citests +Help for rcc pull subcommand 0 build/rcc pull -h --controller citests +Help for rcc robot subcommand 0 build/rcc robot -h --controller citests +Help for rcc run subcommand 0 build/rcc run -h --controller citests +Help for rcc task subcommand 0 build/rcc task -h --controller citests +Help for rcc tutorial subcommand 0 build/rcc tutorial -h --controller citests +Help for rcc version subcommand 0 build/rcc version -h --controller citests +Run rcc config settings 0 build/rcc config settings --controller citests +Run rcc docs changelog 0 build/rcc docs changelog --controller citests +Run rcc docs license 0 build/rcc docs license --controller citests +Run rcc docs recipes 0 build/rcc docs recipes --controller citests +Run rcc docs tutorial 0 build/rcc docs tutorial --controller citests +Run rcc holotree list 0 build/rcc holotree list --controller citests +Run rcc tutorial 0 build/rcc tutorial --controller citests +Run rcc version 0 build/rcc version --controller citests +Run rcc --version 0 build/rcc --version --controller citests -Help for rcc env delete 0 build/rcc env delete -h --controller citests -Help for rcc env list 0 build/rcc env list -h --controller citests -Help for rcc env new 0 build/rcc env new -h --controller citests -Help for rcc env variables 0 build/rcc env variables -h --controller citests - -Help for rcc configure identity 0 build/rcc configure identity -h --controller citests -Help for rcc feedback metric 0 build/rcc feedback metric -h --controller citests - -Help for rcc man license 0 build/rcc man license -h --controller citests - -Help for rcc robot fix 0 build/rcc robot fix -h --controller citests -Help for rcc robot initialize 0 build/rcc robot initialize -h --controller citests -Help for rcc robot libs 0 build/rcc robot libs -h --controller citests -Help for rcc robot list 0 build/rcc robot list -h --controller citests -Help for rcc robot unwrap 0 build/rcc robot unwrap -h --controller citests -Help for rcc robot wrap 0 build/rcc robot wrap -h --controller citests - -Help for rcc task run 0 build/rcc task run -h --controller citests -Help for rcc task shell 0 build/rcc task shell -h --controller citests -Help for rcc task testrun 0 build/rcc task testrun -h --controller citests *** Keywords *** - Verify exitcodes - [Arguments] ${exitcode} ${command} - ${code} ${output}= Run and return rc and output ${command} - Log 
${output}
html=yes - Should be equal as strings ${exitcode} ${code} + [Arguments] ${exitcode} ${command} + ${code} ${output} ${error}= Run and return code output error ${command} + Log STDOUT
${output}
html=yes + Log STDERR
${error}
html=yes + Should be equal as strings ${exitcode} ${code} diff --git a/robot_tests/export_holozip.robot b/robot_tests/export_holozip.robot new file mode 100644 index 00000000..9e30b6e9 --- /dev/null +++ b/robot_tests/export_holozip.robot @@ -0,0 +1,101 @@ +*** Settings *** +Library OperatingSystem +Library supporting.py +Resource resources.robot +Suite Setup Export setup +Suite Teardown Export teardown + +*** Keywords *** +Export setup + Remove Directory tmp/developer True + Remove Directory tmp/guest True + Remove Directory tmp/standalone True + Prepare Robocorp Home tmp/developer + Fire And Forget build/rcc ht delete 4e67cd8 + +Export teardown + Prepare Robocorp Home tmp/robocorp + Remove Directory tmp/developer True + Remove Directory tmp/guest True + Remove Directory tmp/standalone True + +*** Test cases *** + +Goal: Create extended robot into tmp/standalone folder using force. + Step build/rcc robot init --controller citests -t extended -d tmp/standalone -f + Use STDERR + Must Have OK. + + ${output}= Capture Flat Output build/rcc ht hash --silent tmp/standalone/conda.yaml + Set Suite Variable ${fingerprint} ${output} + +Goal: Create environment for standalone robot + Step build/rcc ht vars -s author --controller citests -r tmp/standalone/robot.yaml + Must Have RCC_ENVIRONMENT_HASH= + Must Have RCC_INSTALLATION_ID= + Must Have 4e67cd8_fcb4b859 + Use STDERR + Must Have Progress: 01/15 + Must Have Progress: 15/15 + +Goal: Must have author space visible + Step build/rcc ht ls + Use STDERR + Must Have 4e67cd8_fcb4b859 + Must Have rcc.citests + Must Have author + Must Have ${fingerprint} + Wont Have guest + +Goal: Show exportable environment list + Step build/rcc ht export + Use STDERR + Must Have Selectable catalogs + Must Have - ${fingerprint} + Must Have OK. + +Goal: Export environment for standalone robot + Step build/rcc ht export -z tmp/standalone/hololib.zip ${fingerprint} + Use STDERR + Wont Have Selectable catalogs + Must Have OK. + +Goal: Wrap the robot + Step build/rcc robot wrap -z tmp/full.zip -d tmp/standalone/ + Use STDERR + Must Have OK. + +Goal: See contents of that robot + Step unzip -v tmp/full.zip + Must Have robot.yaml + Must Have conda.yaml + Must Have hololib.zip + +Goal: Can delete author space + Step build/rcc ht delete 4e67cd8_fcb4b859 + Step build/rcc ht ls + Use STDERR + Wont Have 4e67cd8_fcb4b859 + Wont Have rcc.citests + Wont Have author + Wont Have ${fingerprint} + Wont Have guest + +Goal: Can run as guest + Fire And Forget build/rcc ht delete 4e67cd8 + Prepare Robocorp Home tmp/guest + Step build/rcc task run --controller citests -s guest -r tmp/standalone/robot.yaml -t "run example task" + Use STDERR + Must Have point of view, "actual main robot run" was SUCCESS. + Must Have OK. + +Goal: Space created under author for guest + Prepare Robocorp Home tmp/developer + Step build/rcc ht ls + Use STDERR + Wont Have 4e67cd8_fcb4b859 + Wont Have author + Must Have rcc.citests + Must Have ${fingerprint} + Must Have 4e67cd8_aacf1552 + Must Have guest diff --git a/robot_tests/fullrun.robot b/robot_tests/fullrun.robot index d263126a..fa26e992 100644 --- a/robot_tests/fullrun.robot +++ b/robot_tests/fullrun.robot @@ -2,16 +2,29 @@ Library OperatingSystem Library supporting.py Resource resources.robot +Suite Setup Fullrun setup -*** Test cases *** +*** Keywords *** +Fullrun setup + Fire And Forget build/rcc ht delete 4e67cd8 -Using and running template example with shell file +*** Test cases *** - Goal Show rcc version information. +Goal: Show rcc version information. Step build/rcc version --controller citests - Must Have v6. + Must Have v18. + +Goal: There is debug message when bundled case. + Step build/rcc version --controller citests --debug --bundled + Use STDERR + Must Have Did not check newer version existence, since this is bundled case. - Goal Show rcc license information. +Goal: No debug message when user case. + Step build/rcc version --controller citests --debug + Use STDERR + Wont Have this is bundled case + +Goal: Show rcc license information. Step build/rcc man license --controller citests Must Have Apache License Must Have Version 2.0 @@ -19,126 +32,215 @@ Using and running template example with shell file Must Have Copyright 2020 Robocorp Technologies, Inc. Wont Have EULA - Goal Telemetry tracking enabled by default. +Goal: Telemetry tracking enabled by default. Step build/rcc configure identity --controller citests Must Have anonymous health tracking is: enabled Must Exist %{ROBOCORP_HOME}/rcc.yaml - Wont Exist %{ROBOCORP_HOME}/rcccache.yaml - Goal Send telemetry data to cloud. +Goal: Send telemetry data to cloud. Step build/rcc feedback metric --controller citests -t test -n rcc.test -v robot.fullrun + Use STDERR Must Have OK - Goal Telemetry tracking can be disabled. +Goal: Telemetry tracking can be disabled. Step build/rcc configure identity --controller citests --do-not-track Must Have anonymous health tracking is: disabled - Goal Show listing of rcc commands. +Goal: Show listing of rcc commands. Step build/rcc --controller citests + Use STDERR Must Have rcc is environment manager Wont Have missing - Goal Show toplevel help for rcc. +Goal: Show toplevel help for rcc. Step build/rcc -h Must Have Available Commands: - Goal Show config help for rcc. +Goal: Show config help for rcc. Step build/rcc config -h --controller citests Must Have Available Commands: Must Have credentials - Goal List available robot templates. - Step build/rcc robot init -l --controller citests +Goal: List available robot templates. + Step build/rcc robot init -i -l --controller citests Must Have extended Must Have python Must Have standard + Use STDERR Must Have OK. - Goal Initialize new standard robot into tmp/fluffy folder using force. - Step build/rcc robot init --controller citests -t extended -d tmp/fluffy -f +Goal: Initialize new standard robot into tmp/fluffy folder using force. + Step build/rcc robot init -i --controller citests -t extended -d tmp/fluffy -f + Use STDERR Must Have OK. - Goal There should now be fluffy in robot listing - Step build/rcc robot list --controller citests -j - Must Be Json Response - Must Have fluffy - Must Have "robot" - - Goal Fail to initialize new standard robot into tmp/fluffy without force. - Step build/rcc robot init --controller citests -t extended -d tmp/fluffy 2 +Goal: Fail to initialize new standard robot into tmp/fluffy without force. + Step build/rcc robot init -i --controller citests -t extended -d tmp/fluffy 2 + Use STDERR Must Have Error: Directory Must Have fluffy is not empty + Wont Exist tmp/fluffy/output/environment_*_freeze.yaml - Goal Run task in place. - Step build/rcc task run --controller citests -r tmp/fluffy/robot.yaml - Must Have Progress: 0/4 - Must Have Progress: 4/4 +Goal: Run task in place in debug mode and with timeline. + Step build/rcc task run --task "Run Example task" --controller citests -r tmp/fluffy/robot.yaml --debug --timeline + Must Have 1 task, 1 passed, 0 failed + Use STDERR + Must Have Progress: 01/15 + Must Have Progress: 02/15 + Must Have Progress: 15/15 Must Have rpaframework - Must Have 1 critical task, 1 passed, 0 failed + Must Have PID # + Must Have [N] + Must Have [D] + Wont Have [T] + Wont Have Running against old environment + Wont Have WARNING + Wont Have NOT pristine + Must Have Installation plan is: + Must Have Command line is: [ + Must Have rcc timeline + Must Have robot execution (simple=false). + Must Have Now. + Must Have Wanted + Must Have Available + Must Have Version + Must Have Origin + Must Have Status + Must Have point of view, "actual main robot run" was SUCCESS. Must Have OK. - Must Exist %{ROBOCORP_HOME}/base/ - Must Exist %{ROBOCORP_HOME}/live/ + Must Exist tmp/fluffy/output/environment_*_freeze.yaml Must Exist %{ROBOCORP_HOME}/wheels/ Must Exist %{ROBOCORP_HOME}/pipcache/ + Step build/rcc holotree check --controller citests - Goal Run task in clean temporary directory. - Step build/rcc task testrun --controller citests -r tmp/fluffy/robot.yaml - Must Have Progress: 0/4 - Wont Have Progress: 1/4 - Wont Have Progress: 2/4 - Wont Have Progress: 3/4 - Must Have Progress: 4/4 +Goal: Run task in clean temporary directory. + Step build/rcc task testrun --task "Run Example task" --controller citests -r tmp/fluffy/robot.yaml + Must Have 1 task, 1 passed, 0 failed + Use STDERR Must Have rpaframework - Must Have 1 critical task, 1 passed, 0 failed + Must Have Progress: 01/15 + Wont Have Progress: 03/15 + Wont Have Progress: 05/15 + Wont Have Progress: 07/15 + Wont Have Progress: 09/15 + Must Have Progress: 14/15 + Must Have Progress: 15/15 + Must Have point of view, "actual main robot run" was SUCCESS. Must Have OK. - Goal Merge two different conda.yaml files with conflict fails - Step build/rcc env new --controller citests conda/testdata/conda.yaml conda/testdata/other.yaml 1 +Goal: Merge two different conda.yaml files with conflict fails + Step build/rcc holotree vars --controller citests conda/testdata/conda.yaml conda/testdata/other.yaml 5 + Use STDERR Must Have robotframework=3.1 vs. robotframework=3.2 - Goal Merge two different conda.yaml files with conflict fails - Step build/rcc env new --controller citests conda/testdata/other.yaml conda/testdata/third.yaml --silent - Must Have 44d08c86724dd710b33228c3a1ea1a0b4bef1a44f01fe825d0e412044aaa7c0242030a +Goal: Merge two different conda.yaml files without conflict passes + Step build/rcc holotree vars --controller citests conda/testdata/third.yaml conda/testdata/other.yaml --silent + Must Have RCC_ENVIRONMENT_HASH=ffd32af1fdf0f253 + Must Have 4e67cd8_9fcd2534 - Goal See variables from specific environment without robot.yaml knowledge - Step build/rcc env variables --controller citests conda/testdata/conda.yaml +Goal: Can list environments as JSON + Step build/rcc holotree list --controller citests --json + Must Have 4e67cd8_9fcd2534 + Must Have ffd32af1fdf0f253 + Must Be Json Response + +Goal: See variables from specific environment without robot.yaml knowledge + Step build/rcc holotree variables --controller citests conda/testdata/conda.yaml Must Have ROBOCORP_HOME= Must Have PYTHON_EXE= + Must Have RCC_EXE= Must Have CONDA_DEFAULT_ENV=rcc - Must Have CONDA_EXE= Must Have CONDA_PREFIX= Must Have CONDA_PROMPT_MODIFIER=(rcc) - Must Have CONDA_PYTHON_EXE= Must Have CONDA_SHLVL=1 Must Have PATH= - Must Have PYTHONPATH= Must Have PYTHONHOME= Must Have PYTHONEXECUTABLE= Must Have PYTHONNOUSERSITE=1 + Must Have TEMP= + Must Have TMP= + Must Have RCC_ENVIRONMENT_HASH= + Must Have RCC_INSTALLATION_ID= + Must Have RCC_TRACKING_ALLOWED= + Wont Have PYTHONPATH= Wont Have ROBOT_ROOT= Wont Have ROBOT_ARTIFACTS= - Must Have ded08c86224cc710b22228d3a1aa1a074bdf1a44f01be819c0a816044eebb80242030a + Must Have RCC_ENVIRONMENT_HASH=786fd9dca1e1f1db + Step build/rcc holotree check --controller citests - Goal See variables from specific environment without robot.yaml knowledge in JSON form - Step build/rcc env variables --controller citests --json conda/testdata/conda.yaml +Goal: See variables from specific environment with robot.yaml but without task + Step build/rcc holotree variables --controller citests -r tmp/fluffy/robot.yaml + Must Have ROBOCORP_HOME= + Must Have PYTHON_EXE= + Must Have RCC_EXE= + Must Have CONDA_DEFAULT_ENV=rcc + Must Have CONDA_PREFIX= + Must Have CONDA_PROMPT_MODIFIER=(rcc) + Must Have CONDA_SHLVL=1 + Must Have PATH= + Must Have PYTHONHOME= + Must Have PYTHONEXECUTABLE= + Must Have PYTHONNOUSERSITE=1 + Must Have TEMP= + Must Have TMP= + Must Have RCC_ENVIRONMENT_HASH=1cdd0b852854fe5b + Must Have RCC_INSTALLATION_ID= + Must Have RCC_TRACKING_ALLOWED= + Must Have PYTHONPATH= + Must Have ROBOT_ROOT= + Must Have ROBOT_ARTIFACTS= + Step build/rcc holotree check --controller citests + +Goal: See variables from specific environment with warranty voided + Step build/rcc holotree variables --controller citests -r tmp/fluffy/robot.yaml --warranty-voided --anything I_know_what_Im_doing + Must Have ROBOCORP_HOME= + Must Have PYTHON_EXE= + Must Have RCC_EXE= + Must Have CONDA_DEFAULT_ENV=rcc + Must Have CONDA_PREFIX= + Must Have CONDA_PROMPT_MODIFIER=(rcc) + Must Have CONDA_SHLVL=1 + Must Have PATH= + Must Have PYTHONHOME= + Must Have PYTHONEXECUTABLE= + Must Have PYTHONNOUSERSITE=1 + Must Have TEMP= + Must Have TMP= + Must Have RCC_ENVIRONMENT_HASH= + Must Have RCC_INSTALLATION_ID= + Must Have RCC_TRACKING_ALLOWED= + Must Have PYTHONPATH= + Must Have ROBOT_ROOT= + Must Have ROBOT_ARTIFACTS= + Use STDERR + Wont Have Progress: 01/15 + Wont Have Progress: 02/15 + Wont Have Progress: 15/15 + Must Have Warning: Note that 'rcc' is running in 'warranty voided' mode. + +Goal: See variables from specific environment without robot.yaml knowledge in JSON form + Step build/rcc holotree variables --controller citests --json conda/testdata/conda.yaml Must Be Json Response - Goal See variables from specific environment with robot.yaml knowledge - Step build/rcc env variables --controller citests conda/testdata/conda.yaml --config tmp/alternative.yaml -r tmp/fluffy/robot.yaml -e tmp/fluffy/devdata/env.json +Goal: See variables from specific environment with robot.yaml knowledge + Step build/rcc holotree variables --controller citests conda/testdata/conda.yaml --config tmp/alternative.yaml -r tmp/fluffy/robot.yaml -e tmp/fluffy/devdata/env.json Must Have ROBOCORP_HOME= Must Have PYTHON_EXE= - Must Have THE_ANSWER=42 + Must Have RCC_EXE= Must Have CONDA_DEFAULT_ENV=rcc - Must Have CONDA_EXE= Must Have CONDA_PREFIX= Must Have CONDA_PROMPT_MODIFIER=(rcc) - Must Have CONDA_PYTHON_EXE= Must Have CONDA_SHLVL=1 Must Have PATH= Must Have PYTHONPATH= Must Have PYTHONHOME= Must Have PYTHONEXECUTABLE= Must Have PYTHONNOUSERSITE=1 + Must Have TEMP= + Must Have TMP= + Must Have RCC_ENVIRONMENT_HASH= + Must Have RCC_INSTALLATION_ID= + Must Have RCC_TRACKING_ALLOWED= Must Have ROBOT_ROOT= Must Have ROBOT_ARTIFACTS= Wont Have RC_API_SECRET_HOST= @@ -146,8 +248,22 @@ Using and running template example with shell file Wont Have RC_API_SECRET_TOKEN= Wont Have RC_API_WORKITEM_TOKEN= Wont Have RC_WORKSPACE_ID= + Step build/rcc holotree check --controller citests + +Goal: See variables from specific environment with robot.yaml knowledge in JSON form + Step build/rcc holotree variables --controller citests --json conda/testdata/conda.yaml --config tmp/alternative.yaml -r tmp/fluffy/robot.yaml -e tmp/fluffy/devdata/env.json + Must Be Json Response - Goal See variables from specific environment with robot.yaml knowledge in JSON form - Step build/rcc env variables --controller citests --json conda/testdata/conda.yaml --config tmp/alternative.yaml -r tmp/fluffy/robot.yaml -e tmp/fluffy/devdata/env.json +Goal: See diagnostics as valid JSON form + Step build/rcc configure diagnostics --json Must Be Json Response +Goal: Simulate issue report sending with dryrun + Step build/rcc feedback issue --controller citests --dryrun --report robot_tests/report.json --attachments robot_tests/conda.yaml + Must Have "report": + Must Have "zipfile": + Must Have "installationId": + Must Have "platform": + Must Be Json Response + Use STDERR + Must Have OK diff --git a/robot_tests/holotree.robot b/robot_tests/holotree.robot new file mode 100644 index 00000000..42f864c9 --- /dev/null +++ b/robot_tests/holotree.robot @@ -0,0 +1,139 @@ +*** Settings *** +Library OperatingSystem +Library supporting.py +Resource resources.robot +Suite Setup Holotree setup + +*** Keywords *** +Holotree setup + Fire And Forget build/rcc ht delete 4e67cd8 + +*** Test cases *** + +Goal: Initialize new standard robot into tmp/holotin folder using force. + Step build/rcc robot init --controller citests -t extended -d tmp/holotin -f + Use STDERR + Must Have OK. + +Goal: See variables from specific environment without robot.yaml knowledge + Step build/rcc holotree variables --space jam --controller citests conda/testdata/conda.yaml + Must Have ROBOCORP_HOME= + Must Have PYTHON_EXE= + Must Have CONDA_DEFAULT_ENV=rcc + Must Have CONDA_PREFIX= + Must Have CONDA_PROMPT_MODIFIER=(rcc) + Must Have CONDA_SHLVL=1 + Must Have PATH= + Must Have PYTHONHOME= + Must Have PYTHONEXECUTABLE= + Must Have PYTHONNOUSERSITE=1 + Must Have TEMP= + Must Have TMP= + Must Have RCC_ENVIRONMENT_HASH= + Must Have RCC_INSTALLATION_ID= + Must Have RCC_TRACKING_ALLOWED= + Wont Have PYTHONPATH= + Wont Have ROBOT_ROOT= + Wont Have ROBOT_ARTIFACTS= + +Goal: See variables from specific environment with robot.yaml but without task + Step build/rcc holotree variables --space holotin --controller citests -r tmp/holotin/robot.yaml + Must Have ROBOCORP_HOME= + Must Have PYTHON_EXE= + Must Have CONDA_DEFAULT_ENV=rcc + Must Have CONDA_PREFIX= + Must Have CONDA_PROMPT_MODIFIER=(rcc) + Must Have CONDA_SHLVL=1 + Must Have PATH= + Must Have PYTHONHOME= + Must Have PYTHONEXECUTABLE= + Must Have PYTHONNOUSERSITE=1 + Must Have TEMP= + Must Have TMP= + Must Have RCC_ENVIRONMENT_HASH= + Must Have RCC_INSTALLATION_ID= + Must Have RCC_TRACKING_ALLOWED= + Must Have PYTHONPATH= + Must Have ROBOT_ROOT= + Must Have ROBOT_ARTIFACTS= + +Goal: See variables from specific environment without robot.yaml knowledge in JSON form + Step build/rcc holotree variables --space jam --controller citests --json conda/testdata/conda.yaml + Must Be Json Response + +Goal: See variables from specific environment with robot.yaml knowledge + Step build/rcc holotree variables --space jam --controller citests conda/testdata/conda.yaml --config tmp/alternative.yaml -r tmp/holotin/robot.yaml -e tmp/holotin/devdata/env.json + Must Have ROBOCORP_HOME= + Must Have PYTHON_EXE= + Must Have CONDA_DEFAULT_ENV=rcc + Must Have CONDA_PREFIX= + Must Have CONDA_PROMPT_MODIFIER=(rcc) + Must Have CONDA_SHLVL=1 + Must Have PATH= + Must Have PYTHONPATH= + Must Have PYTHONHOME= + Must Have PYTHONEXECUTABLE= + Must Have PYTHONNOUSERSITE=1 + Must Have TEMP= + Must Have TMP= + Must Have RCC_ENVIRONMENT_HASH= + Must Have RCC_INSTALLATION_ID= + Must Have RCC_TRACKING_ALLOWED= + Must Have ROBOT_ROOT= + Must Have ROBOT_ARTIFACTS= + Wont Have RC_API_SECRET_HOST= + Wont Have RC_API_WORKITEM_HOST= + Wont Have RC_API_SECRET_TOKEN= + Wont Have RC_API_WORKITEM_TOKEN= + Wont Have RC_WORKSPACE_ID= + Use STDERR + Wont Have (virtual) + +Goal: See variables from specific environment with robot.yaml knowledge in JSON form + Step build/rcc holotree variables --space jam --controller citests --json conda/testdata/conda.yaml --config tmp/alternative.yaml -r tmp/holotin/robot.yaml -e tmp/holotin/devdata/env.json + Must Be Json Response + Use STDERR + Wont Have (virtual) + +Goal: Liveonly works and uses virtual holotree + Step build/rcc holotree vars --liveonly --space jam --controller citests robot_tests/certificates.yaml --config tmp/alternative.yaml --timeline + Must Have ROBOCORP_HOME= + Must Have CONDA_DEFAULT_ENV=rcc + Must Have CONDA_PREFIX= + Must Have CONDA_PROMPT_MODIFIER=(rcc) + Must Have CONDA_SHLVL=1 + Must Have PATH= + Must Have PYTHONHOME= + Must Have PYTHONEXECUTABLE= + Must Have PYTHONNOUSERSITE=1 + Must Have TEMP= + Must Have TMP= + Must Have RCC_ENVIRONMENT_HASH= + Must Have RCC_INSTALLATION_ID= + Must Have RCC_TRACKING_ALLOWED= + Wont Have PYTHON_EXE= + Wont Have PYTHONPATH= + Wont Have ROBOT_ROOT= + Wont Have ROBOT_ARTIFACTS= + Wont Have RC_API_SECRET_HOST= + Wont Have RC_API_WORKITEM_HOST= + Wont Have RC_API_SECRET_TOKEN= + Wont Have RC_API_WORKITEM_TOKEN= + Wont Have RC_WORKSPACE_ID= + Use STDERR + Must Have (virtual) + +Goal: Do quick cleanup on environments + Step build/rcc config cleanup --controller citests --quick + Must Exist %{ROBOCORP_HOME}/micromamba/ + Wont Exist %{ROBOCORP_HOME}/pkgs/ + Wont Exist %{ROBOCORP_HOME}/pipcache/ + Wont Exist %{ROBOCORP_HOME}/templates/ + Use STDERR + Must Have OK + +Goal: Liveonly works and uses virtual holotree and can give output in JSON form + Step build/rcc ht vars --liveonly --space jam --controller citests --json robot_tests/certificates.yaml --config tmp/alternative.yaml --timeline + Must Be Json Response + Use STDERR + Must Have (virtual) diff --git a/robot_tests/maintenance.robot b/robot_tests/maintenance.robot new file mode 100644 index 00000000..6643781e --- /dev/null +++ b/robot_tests/maintenance.robot @@ -0,0 +1,52 @@ +*** Settings *** +Library OperatingSystem +Library supporting.py +Resource resources.robot + +*** Test cases *** + +Goal: Can see human readable catalog of robots + Step build/rcc holotree catalogs --controller citests + Use STDERR + Must Have inside hololib + Must Have Age (days) + Must Have Idle (days) + Must Have ffd32af1fdf0f253 + Must Have OK. + +Goal: Can see machine readable catalog of robots + Step build/rcc holotree catalogs --controller citests --json + Must Be Json Response + Must Have ffd32af1fdf0f253 + Must Have "age_in_days": 0, + Must Have "days_since_last_use": 0, + Must Have "holotree": + Must Have "blueprint": + Must Have "files": + Must Have "directories": + +Goal: Can check holotree with retries + Step build/rcc holotree check --retries 5 --controller citests + Use STDERR + Must Have OK. + +Goal: Can remove catalogs with check from hololib by ids and give warnings + Step build/rcc holotree remove cafebabe9000 --check 5 --controller citests + Use STDERR + Must Have Warning: No catalogs given, so nothing to do. Quitting! + Wont Have Warning: Remember to run `rcc holotree check` after you have removed all desired catalogs! + Must Have OK. + +Goal: Can remove catalogs from hololib by idle days and give warnings + Step build/rcc holotree remove --unused 90 --controller citests + Use STDERR + Must Have Warning: No catalogs given, so nothing to do. Quitting! + Must Have Warning: Remember to run `rcc holotree check` after you have removed all desired catalogs! + Must Have OK. + +Goal: Can remove catalogs with check from hololib by ids correctly + Step build/rcc holotree remove ffd32af1fdf0f253 --check 5 --controller citests + Use STDERR + Wont Have Warning: No catalogs given, so nothing to do. Quitting! + Wont Have Warning: Remember to run `rcc holotree check` after you have removed all desired catalogs! + Must Have OK. diff --git a/robot_tests/profile_alpha.yaml b/robot_tests/profile_alpha.yaml new file mode 100644 index 00000000..8c8b26d4 --- /dev/null +++ b/robot_tests/profile_alpha.yaml @@ -0,0 +1,14 @@ +name: Alpha +description: Alpha settings +settings: + certificates: + verify-ssl: true + ssl-no-revoke: false + network: + https-proxy: "" + http-proxy: "" + meta: + name: Alpha + description: Alpha settings + source: tests + version: 1.2.3.4 diff --git a/robot_tests/profile_beta.yaml b/robot_tests/profile_beta.yaml new file mode 100644 index 00000000..a9bf2c38 --- /dev/null +++ b/robot_tests/profile_beta.yaml @@ -0,0 +1,21 @@ +name: Beta +description: Beta settings +settings: + certificates: + verify-ssl: false + ssl-no-revoke: true + legacy-renegotiation-allowed: true + network: + no-proxy: noproxy.betaputkinen.net + https-proxy: http://bad.betaputkinen.net:1234/ + http-proxy: http://bad.betaputkinen.net:2345/ + meta: + name: Beta + description: Beta settings + source: tests + version: 1.2.3.4 +piprc: | + [global] + trusted-host = pypi.python.org pypi.org files.pythonhosted.org www.googleapis.com api.github.com selenium-release.storage.googleapis.com +micromambarc: | + ssl_verify: False diff --git a/robot_tests/profile_gamma.yaml b/robot_tests/profile_gamma.yaml new file mode 100644 index 00000000..3eb32489 --- /dev/null +++ b/robot_tests/profile_gamma.yaml @@ -0,0 +1,14 @@ +name: Gamma +description: Gamma settings +settings: + certificates: + verify-ssl: true + ssl-no-revoke: false + network: + https-proxy: "" + http-proxy: "" + meta: + name: Gamma + description: Gamma settings + source: tests + version: 1.2.3.4 diff --git a/robot_tests/profiles.robot b/robot_tests/profiles.robot new file mode 100644 index 00000000..8c9c072a --- /dev/null +++ b/robot_tests/profiles.robot @@ -0,0 +1,128 @@ +*** Settings *** +Library OperatingSystem +Library supporting.py +Resource resources.robot + +*** Test cases *** + +Goal: Initially there are no profiles + Step build/rcc configuration switch 2 + Use STDERR + Must Have No profiles found, you must first import some. + +Goal: Can import profiles into rcc + Step build/rcc configuration import --filename robot_tests/profile_alpha.yaml + Use STDERR + Must Have OK. + + Step build/rcc configuration import --filename robot_tests/profile_beta.yaml + Use STDERR + Must Have OK. + +Goal: Can see imported profiles + Step build/rcc configuration switch + Must Have Alpha: Alpha settings + Must Have Beta: Beta settings + Wont Have Gamma: Gamma settings + Must Have Currently active profile is: default + Use STDERR + Must Have OK. + +Goal: Can see imported profiles as json + Step build/rcc configuration switch --json + Must Be Json Response + Must Have "current" + Must Have "profiles" + Must Have "Alpha settings" + Must Have "Beta settings" + +Goal: Can switch to Alpha profile + Step build/rcc configuration switch --profile alpha + Use STDERR + Must Have OK. + + Step build/rcc configuration switch + Use STDOUT + Must Have Currently active profile is: Alpha + Use STDERR + Must Have OK. + +Goal: Quick diagnostics can show alpha profile information + Step build/rcc configuration diagnostics --quick --json + Must Be Json Response + Must Have "config-micromambarc-used": "false" + Must Have "config-piprc-used": "false" + Must Have "config-settings-yaml-used": "true" + Must Have "config-ssl-no-revoke": "false" + Must Have "config-ssl-verify": "true" + Must Have "config-no-proxy": "" + Must Have "config-https-proxy": "" + Must Have "config-http-proxy": "" + +Goal: Can switch to Beta profile + Step build/rcc configuration switch --profile Beta + Use STDERR + Must Have OK. + + Step build/rcc configuration switch + Use STDOUT + Must Have Currently active profile is: Beta + Use STDERR + Must Have OK. + +Goal: Quick diagnostics can show beta profile information + Step build/rcc configuration diagnostics --quick --json + Must Be Json Response + Must Have "config-micromambarc-used": "true" + Must Have "config-piprc-used": "true" + Must Have "config-settings-yaml-used": "true" + Must Have "config-ssl-no-revoke": "true" + Must Have "config-ssl-verify": "false" + Must Have "config-legacy-renegotiation-allowed": "true" + Must Have "config-no-proxy": "noproxy.betaputkinen.net" + Must Have "config-https-proxy": "http://bad.betaputkinen.net:1234/" + Must Have "config-http-proxy": "http://bad.betaputkinen.net:2345/" + +Goal: Can import and switch to Gamma profile immediately + Step build/rcc configuration import --filename robot_tests/profile_gamma.yaml --switch + Use STDERR + Must Have OK. + + Step build/rcc configuration switch + Use STDOUT + Must Have Alpha: Alpha settings + Must Have Beta: Beta settings + Must Have Gamma: Gamma settings + Must Have Currently active profile is: Gamma + Use STDERR + Must Have OK. + +Goal: Can remove profile while it is still used + Step build/rcc configuration remove --profile Gamma + Use STDERR + Must Have OK. + + Step build/rcc configuration switch + Use STDOUT + Must Have Alpha: Alpha settings + Must Have Beta: Beta settings + Wont Have Gamma: Gamma settings + Must Have Currently active profile is: Gamma + Use STDERR + Must Have OK. + +Goal: Can switch to no profile + Step build/rcc configuration switch --noprofile + Use STDERR + Must Have OK. + + Step build/rcc configuration switch + Use STDOUT + Must Have Currently active profile is: default + Use STDERR + Must Have OK. + +Goal: Can export profile + Step build/rcc configuration export --profile Alpha --filename tmp/exported_alpha.yaml + Use STDERR + Must Have OK. diff --git a/robot_tests/python375.yaml b/robot_tests/python375.yaml new file mode 100644 index 00000000..514f9dd5 --- /dev/null +++ b/robot_tests/python375.yaml @@ -0,0 +1,7 @@ +channels: +- conda-forge +dependencies: +- python==3.7.5 +- pip==20.1 +- pip: + - requests==2.28.1 diff --git a/robot_tests/python3913.yaml b/robot_tests/python3913.yaml new file mode 100644 index 00000000..3196015f --- /dev/null +++ b/robot_tests/python3913.yaml @@ -0,0 +1,7 @@ +channels: +- conda-forge +dependencies: +- python==3.9.13 +- pip==22.2.2 +- pip: + - requests==2.28.1 diff --git a/robot_tests/report.json b/robot_tests/report.json new file mode 100644 index 00000000..67a84e02 --- /dev/null +++ b/robot_tests/report.json @@ -0,0 +1,5 @@ +{ + "errorCode": "test", + "errorName": "robot test", + "dialogMessage": "delete this, this it just a test issue (with attachment)" +} diff --git a/robot_tests/resources.robot b/robot_tests/resources.robot index 0d710b3c..1d444d14 100644 --- a/robot_tests/resources.robot +++ b/robot_tests/resources.robot @@ -7,35 +7,62 @@ Library supporting.py Clean Local Remove Directory tmp/robocorp True +Prepare Sema4.ai Home + [Arguments] ${location} + Create Directory ${location} + Remove Environment Variable ROBOCORP_HOME + Set Environment Variable SEMA4AI_HOME ${location} + Copy File robot_tests/settings.yaml ${location}/settings.yaml + Fire And Forget build/rcc --sema4ai ht init --revoke --controller citests + +Prepare Robocorp Home + [Arguments] ${location} + Create Directory ${location} + Remove Environment Variable SEMA4AI_HOME + Set Environment Variable ROBOCORP_HOME ${location} + Copy File robot_tests/settings.yaml ${location}/settings.yaml + Prepare Local Remove Directory tmp/fluffy True Remove Directory tmp/nodogs True Remove Directory tmp/robocorp True Remove File tmp/nodogs.zip - Create Directory tmp/robocorp - Set Environment Variable ROBOCORP_HOME tmp/robocorp - - Goal Verify miniconda is installed or download and install it. - Step build/rcc conda check -i - Must Have OK. - Must Exist %{ROBOCORP_HOME}/miniconda3/ - Wont Exist %{ROBOCORP_HOME}/base/ - Wont Exist %{ROBOCORP_HOME}/live/ - Wont Exist %{ROBOCORP_HOME}/wheels/ - Wont Exist %{ROBOCORP_HOME}/pipcache/ - -Goal - [Arguments] ${anything} - Comment ${anything} + Prepare Robocorp Home tmp/robocorp + + Comment Make sure that tests do not use shared holotree + Fire And Forget build/rcc ht init --revoke + + Fire And Forget build/rcc ht delete 4e67cd8 + + Comment Verify micromamba is installed or download and install it. + Step build/rcc ht vars --controller citests robot_tests/conda.yaml + Must Exist %{ROBOCORP_HOME}/bin/ + Must Exist %{ROBOCORP_HOME}/wheels/ + Must Exist %{ROBOCORP_HOME}/pipcache/ + +Fire And Forget + [Arguments] ${command} + ${code} ${output} ${error}= Run and return code output error ${command} + Log STDOUT
${output}
html=yes + Log STDERR
${error}
html=yes Step [Arguments] ${command} ${expected}=0 - ${code} ${output}= Run and return rc and output ${command} - Set Suite Variable ${robot_output} ${output} - Log 
${output}
html=yes + ${code} ${output} ${error}= Run and return code output error ${command} + Set Suite Variable ${robot_stdout} ${output} + Set Suite Variable ${robot_stderr} ${error} + Use Stdout + Log STDOUT
${output}
html=yes + Log STDERR
${error}
html=yes Should be equal as strings ${expected} ${code} Wont Have Failure: +Use Stdout + Set Suite Variable ${robot_output} ${robot_stdout} + +Use Stderr + Set Suite Variable ${robot_output} ${robot_stderr} + Must Be [Arguments] ${content} Should Be Equal As Strings ${robot_output} ${content} diff --git a/robot_tests/sema4ai.robot b/robot_tests/sema4ai.robot new file mode 100644 index 00000000..7624f3a2 --- /dev/null +++ b/robot_tests/sema4ai.robot @@ -0,0 +1,70 @@ +*** Settings *** +Library OperatingSystem +Library supporting.py +Resource resources.robot +Suite Setup Sema4.ai setup +Suite Teardown Sema4.ai teardown +Default tags WIP + +*** Keywords *** +Sema4.ai setup + Remove Directory tmp/sema4home True + Prepare Sema4.ai Home tmp/sema4home + +Sema4.ai teardown + Remove Directory tmp/sema4home True + Prepare Robocorp Home tmp/robocorp + +*** Test cases *** + +Goal: See rcc toplevel help for Sema4.ai + Step build/rcc --sema4ai --controller citests --help + Must Have SEMA4AI + Must Have Robocorp + Must Have --robocorp + Must Have --sema4ai + Must Have completion + Must Have robot + Wont Have ROBOCORP + Wont Have Robot + Wont Have assistant + Wont Have interactive + Wont Have community + Wont Have tutorial + Wont Have bash + Wont Have fish + +Goal: See rcc commands for Sema4.ai + Step build/rcc --sema4ai --controller citests + Use STDERR + Must Have SEMA4AI + Wont Have ROBOCORP + Wont Have Robocorp + Wont Have Robot + Must Have robot + Wont Have assistant + Wont Have interactive + Wont Have community + Wont Have tutorial + Wont Have completion + Wont Have bash + Wont Have fish + +Goal: Default settings.yaml for Sema4.ai + Step build/rcc --sema4ai configuration settings --defaults --controller citests + Must Have Sema4.ai default settings.yaml + Wont Have assistant + Wont Have branding + Wont Have logo + +Goal: Create package.yaml environment using uv + Step build/rcc --sema4ai ht vars -s sema4ai --controller citests robot_tests/bare_action/package.yaml + Must Have RCC_ENVIRONMENT_HASH= + Must Have RCC_INSTALLATION_ID= + Must Have SEMA4AI_HOME= + Wont Have ROBOCORP_HOME= + Must Have _4e67cd8_81359368 + Use STDERR + Must Have Progress: 01/15 + Must Have Progress: 15/15 + Must Have Running uv install phase. diff --git a/robot_tests/settings.yaml b/robot_tests/settings.yaml new file mode 100644 index 00000000..ac2b3098 --- /dev/null +++ b/robot_tests/settings.yaml @@ -0,0 +1,2 @@ +autoupdates: + templates: https://downloads.robocorp.com/templates/not_available.yaml diff --git a/robot_tests/spellbug/conda.yaml b/robot_tests/spellbug/conda.yaml new file mode 100644 index 00000000..7b0bedcb --- /dev/null +++ b/robot_tests/spellbug/conda.yaml @@ -0,0 +1,7 @@ +channels: +- conda-forge +dependencies: +- python=3.9.13 +- pip=22.1.2 +- pip: + - pyspellchecker==0.6.2 diff --git a/robot_tests/spellbug/robot.yaml b/robot_tests/spellbug/robot.yaml new file mode 100644 index 00000000..57971295 --- /dev/null +++ b/robot_tests/spellbug/robot.yaml @@ -0,0 +1,14 @@ +tasks: + entrypoint: + command: + - python + - task.py + +condaConfigFile: conda.yaml +artifactsDir: output +PATH: + - . +PYTHONPATH: + - . +ignoreFiles: + - .gitignore diff --git a/robot_tests/spellbug/task.py b/robot_tests/spellbug/task.py new file mode 100755 index 00000000..4429e3db --- /dev/null +++ b/robot_tests/spellbug/task.py @@ -0,0 +1,4 @@ +from spellchecker import SpellChecker + +SpellChecker() +print("Bug fixed!") diff --git a/robot_tests/supporting.py b/robot_tests/supporting.py index 9cb97652..ca15586e 100644 --- a/robot_tests/supporting.py +++ b/robot_tests/supporting.py @@ -1,6 +1,77 @@ import json +import logging +import subprocess +import sys + +log = logging.getLogger(__name__) + + +def fix_command(command): + if sys.platform == "win32": + command = command.replace("build/rcc", ".\\build\\rcc.exe", 1) + return command + + +def get_cwd(): + import os + + cwd = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + detail = ( + "(rcc doesn't seem to be built, please run `inv local` before running tests)" + ) + assert "build" in os.listdir(cwd), f"Missing build directory in: {cwd!r} {detail}" + + build_dir = os.path.join(cwd, "build") + if sys.platform == "win32": + assert "rcc.exe" in os.listdir( + build_dir + ), f"Missing rcc.exe in: {build_dir!r} {detail}" + else: + assert "rcc" in os.listdir(build_dir), f"Missing rcc in: {build_dir!r} {detail}" + return cwd + + +def log_command(command: str, cwd: str): + msg = f"Running command: {command!r} cwd: {cwd!r}" + log.info(msg) + + +def capture_flat_output(command): + command = fix_command(command) + cwd = get_cwd() + log_command(command, cwd) + + task = subprocess.Popen( + command, + shell=True, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + cwd=cwd, + ) + out, _ = task.communicate() + assert ( + task.returncode == 0 + ), f"Unexpected exit code {task.returncode} from {command!r}" + return out.decode().strip() + + +def run_and_return_code_output_error(command): + command = fix_command(command) + cwd = get_cwd() + log_command(command, cwd) + + task = subprocess.Popen( + command, + shell=True, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + cwd=cwd, + ) + out, err = task.communicate() + return task.returncode, out.decode(), err.decode() + def parse_json(content): parsed = json.loads(content) - assert isinstance(parsed, (list, dict)), f'Expecting list or dict; got {parsed!r}' + assert isinstance(parsed, (list, dict)), f"Expecting list or dict; got {parsed!r}" return parsed diff --git a/robot_tests/templates.robot b/robot_tests/templates.robot new file mode 100644 index 00000000..c1f0bdf3 --- /dev/null +++ b/robot_tests/templates.robot @@ -0,0 +1,79 @@ +*** Settings *** +Library OperatingSystem +Library supporting.py +Resource resources.robot + +*** Test cases *** + +Goal: Initialize new standard robot. + Step build/rcc robot init -i --controller citests -t standard -d tmp/standardi -f + Use STDERR + Must Have OK. + +Goal: Standard robot has correct hash. + Step build/rcc holotree hash --silent --controller citests tmp/standardi/conda.yaml + Must Have 1cdd0b852854fe5b + +Goal: Running standard robot is succesful. + Step build/rcc task run --space templates --controller citests --robot tmp/standardi/robot.yaml + Use STDERR + Must Have point of view, "actual main robot run" was SUCCESS. + Must Have OK. + +Goal: Initialize new python robot. + Step build/rcc robot init -i --controller citests -t python -d tmp/pythoni -f + Use STDERR + Must Have OK. + +Goal: Python robot has correct hash. + Step build/rcc holotree hash --silent --controller citests tmp/pythoni/conda.yaml + Must Have 1cdd0b852854fe5b + +Goal: Running python robot is succesful. + Step build/rcc task run --space templates --controller citests --robot tmp/pythoni/robot.yaml + Use STDERR + Must Have point of view, "actual main robot run" was SUCCESS. + Must Have OK. + +Goal: Initialize new extended robot. + Step build/rcc robot init -i --controller citests -t extended -d tmp/extendedi -f + Use STDERR + Must Have OK. + +Goal: Extended robot has correct hash. + Step build/rcc holotree hash --silent --controller citests tmp/extendedi/conda.yaml + Must Have 1cdd0b852854fe5b + +Goal: Running extended robot is succesful. (Run All Tasks) + Step build/rcc task run --space templates --task "Run All Tasks" --controller citests --robot tmp/extendedi/robot.yaml + Use STDERR + Must Have point of view, "actual main robot run" was SUCCESS. + Must Have OK. + +Goal: Running extended robot is succesful. (Run Example Task) + Step build/rcc task run --space templates --task "Run Example Task" --controller citests --robot tmp/extendedi/robot.yaml + Use STDERR + Must Have point of view, "actual main robot run" was SUCCESS. + Must Have OK. + +Goal: Correct holotree spaces were created. + Step build/rcc holotree list + Use STDERR + Must Have rcc.citests + Must Have templates + Wont Have rcc.user + +Goal: Can get plan for used environment. + Step build/rcc holotree plan 4e67cd8_c6880905 + Must Have micromamba plan + Must Have pip plan + Must Have post install plan + Must Have activation plan + Must Have installation plan complete + Use STDERR + Must Have OK. + +Goal: Holotree is still correct. + Step build/rcc holotree check --controller citests + Use STDERR + Must Have OK. diff --git a/robot_tests/unmanaged_space.robot b/robot_tests/unmanaged_space.robot new file mode 100644 index 00000000..c31c9bd0 --- /dev/null +++ b/robot_tests/unmanaged_space.robot @@ -0,0 +1,132 @@ +*** Settings *** +Library OperatingSystem +Library supporting.py +Resource resources.robot +Suite Setup Holotree setup + +*** Keywords *** +Holotree setup + Fire And Forget build/rcc ht delete 4e67cd8 + +*** Test cases *** + +Goal: See variables from specific unamanged space + Step build/rcc holotree variables --unmanaged --space python39 --controller citests robot_tests/python3913.yaml + Must Have ROBOCORP_HOME= + Must Have PYTHON_EXE= + Must Have CONDA_DEFAULT_ENV=rcc + Must Have CONDA_PREFIX= + Must Have CONDA_PROMPT_MODIFIER=(rcc) + Must Have CONDA_SHLVL=1 + Must Have PATH= + Must Have PYTHONHOME= + Must Have PYTHONEXECUTABLE= + Must Have PYTHONNOUSERSITE=1 + Must Have TEMP= + Must Have TMP= + Must Have RCC_ENVIRONMENT_HASH= + Must Have RCC_INSTALLATION_ID= + Must Have RCC_TRACKING_ALLOWED= + Wont Have PYTHONPATH= + Wont Have ROBOT_ROOT= + Wont Have ROBOT_ARTIFACTS= + Use STDERR + Must Have This is unmanaged holotree space + Must Have Progress: 01/15 + Must Have Progress: 02/15 + Must Have Progress: 04/15 + Must Have Progress: 05/15 + Must Have Progress: 06/15 + Must Have Progress: 14/15 + Must Have Progress: 15/15 + +Goal: Wont allow use of unmanaged space with incompatible conda.yaml + Step build/rcc holotree variables --debug --unmanaged --space python39 --controller citests robot_tests/python375.yaml 6 + Wont Have ROBOCORP_HOME= + Wont Have PYTHON_EXE= + Wont Have RCC_ENVIRONMENT_HASH= + Wont Have RCC_INSTALLATION_ID= + Use STDERR + Must Have This is unmanaged holotree space + Must Have Progress: 01/15 + Must Have Progress: 02/15 + Must Have Progress: 15/15 + + Wont Have Progress: 04/15 + Wont Have Progress: 05/15 + Wont Have Progress: 06/15 + Wont Have Progress: 14/15 + + Must Have Existing unmanaged space fingerprint + Must Have does not match requested one + Must Have Quitting! + +Goal: Allows different unmanaged space for different conda.yaml + Step build/rcc holotree variables --unmanaged --space python37 --controller citests robot_tests/python375.yaml + Must Have ROBOCORP_HOME= + Must Have PYTHON_EXE= + Must Have CONDA_DEFAULT_ENV=rcc + Must Have CONDA_PREFIX= + Must Have CONDA_PROMPT_MODIFIER=(rcc) + Must Have CONDA_SHLVL=1 + Must Have PATH= + Must Have PYTHONHOME= + Must Have PYTHONEXECUTABLE= + Must Have PYTHONNOUSERSITE=1 + Must Have TEMP= + Must Have TMP= + Must Have RCC_ENVIRONMENT_HASH= + Must Have RCC_INSTALLATION_ID= + Must Have RCC_TRACKING_ALLOWED= + Wont Have PYTHONPATH= + Wont Have ROBOT_ROOT= + Wont Have ROBOT_ARTIFACTS= + Use STDERR + Must Have This is unmanaged holotree space + Must Have Progress: 01/15 + Must Have Progress: 02/15 + Must Have Progress: 04/15 + Must Have Progress: 05/15 + Must Have Progress: 06/15 + Must Have Progress: 14/15 + Must Have Progress: 15/15 + +Goal: Wont allow use of unmanaged space with incompatible conda.yaml when two unmanaged spaces exists + Step build/rcc holotree variables --debug --unmanaged --space python37 --controller citests robot_tests/python3913.yaml 6 + Use STDERR + Must Have This is unmanaged holotree space + Must Have Progress: 01/15 + Must Have Progress: 02/15 + Must Have Progress: 15/15 + + Wont Have Progress: 05/15 + Wont Have Progress: 14/15 + + Must Have Existing unmanaged space fingerprint + Must Have does not match requested one + Must Have Quitting! + +Goal: See variables from specific environment without robot.yaml knowledge in JSON form + Step build/rcc holotree variables --unmanaged --space python39 --controller citests --json robot_tests/python3913.yaml + Must Be Json Response + Use STDERR + Must Have This is unmanaged holotree space + +Goal: Can see unmanaged spaces in listings + Step build/rcc holotree list --controller citests + Use STDERR + Must Have UNMNGED_ + Must Have python37 + Must Have python39 + +Goal: Can delete all unmanaged spaces with one command + Step build/rcc holotree delete --controller citests UNMNGED_ + Use STDERR + Must Have Removing UNMNGED_ + +Goal: After deleted, cannot see unmanaged spaces in listings + Step build/rcc holotree list --controller citests + Use STDERR + Wont Have UNMNGED_ + Wont Have python37 + Wont Have python39 diff --git a/scripts/deadcode.py b/scripts/deadcode.py new file mode 100755 index 00000000..21a22df3 --- /dev/null +++ b/scripts/deadcode.py @@ -0,0 +1,57 @@ +#!/bin/env python3 + +import os +import pathlib +import re +import sys +from collections import defaultdict + +FUNC_PATTERN = re.compile(r"^\s*func\s+(\w+)") + + +def read_file(filename): + with open(filename) as source: + for index, line in enumerate(source): + yield index + 1, line + + +def find_files(where, pattern): + return tuple( + sorted(x.relative_to(where) for x in pathlib.Path(where).rglob(pattern)) + ) + + +def find_pattern(pattern, fileset): + for filename in fileset: + for number, line in read_file(filename): + for item in pattern.finditer(line): + yield f"{filename}:{number}", item.group(1) + + +def process(limit): + functions = defaultdict(set) + files = find_files(os.getcwd(), "*.go") + for filename, function in find_pattern(FUNC_PATTERN, files): + functions[function].add(filename) + keys = "|".join(sorted(functions.keys())) + pattern = re.compile(f"({keys})") + counters = defaultdict(int) + linerefs = defaultdict(set) + width = 0 + for fileref, value in find_pattern(pattern, files): + counters[value] += 1 + linerefs[value].add(fileref) + width = max(width, len(fileref)) + for key, value in sorted(counters.items()): + if key.startswith("Test"): + continue + definitions = len(functions[key]) - 1 + if value != limit + definitions: + continue + for link in sorted(linerefs[key]): + fill = " " * (width - len(link)) + print(f"{link}{fill} {key}") + + +if __name__ == "__main__": + process(int(sys.argv[1]) if len(sys.argv) > 1 else 1) diff --git a/scripts/toc.py b/scripts/toc.py new file mode 100755 index 00000000..0527a78f --- /dev/null +++ b/scripts/toc.py @@ -0,0 +1,103 @@ +#!/bin/env python3 + +import glob +import re +from os.path import basename + +DELETE_PATTERN = re.compile(r"[/:]+") +NONCHAR_PATTERN = re.compile(r"[^.a-z0-9_-]+") +HEADING_PATTERN = re.compile(r"^\s*(#{1,3})\s+(.*?)\s*$") +CODE_PATTERN = re.compile(r"^\s*[`]{3}") + +DOT = "." +DASH = "-" +NEWLINE = "\n" + +IGNORE_LIST = ("changelog.md", "toc.md", "BUILD.md", "README.md") + +PRIORITY_LIST = ( + "docs/usecases.md", + "docs/features.md", + "docs/recipes.md", + "docs/profile_configuration.md", + "docs/environment-caching.md", + "docs/maintenance.md", + "docs/venv.md", + "docs/troubleshooting.md", + "docs/vocabulary.md", + "docs/history.md", +) + + +def unify(value): + low = DELETE_PATTERN.sub("", str(value).lower()) + return DASH.join(filter(bool, NONCHAR_PATTERN.split(low))).replace(".", "") + + +class Toc: + def __init__(self, title, baseurl): + self.title = title + self.baseurl = baseurl + self.levels = [0] + self.toc = [f"# {title}"] + + def leveling(self, level): + levelup = True + while len(self.levels) > level: + self.levels.pop() + while len(self.levels) < level: + self.levels.append(1) + levelup = False + if levelup: + self.levels[-1] += 1 + + def add(self, filename, level, title): + self.leveling(level) + numbering = DOT.join(map(str, self.levels)) + url = f"{self.baseurl}{filename}" + prefix = "#" * level + ref = unify(title) + self.toc.append( + f"#{prefix} {numbering} [{title}]({self.baseurl}{filename}#{ref})" + ) + + def write(self, filename): + with open(filename, "w+") as sink: + sink.write(NEWLINE.join(self.toc)) + + +def headings(filename): + inside = False + with open(filename, encoding="utf-8") as source: + for line in source: + if CODE_PATTERN.match(line): + inside = not inside + if inside: + continue + if found := HEADING_PATTERN.match(line): + level, title = found.groups() + yield filename, len(level), title + + +def process(): + toc = Toc( + "Table of contents: rcc documentation", + "https://github.com/robocorp/rcc/blob/master/", + ) + flatnames = list(map(basename, glob.glob("docs/*.md"))) + for filename in PRIORITY_LIST: + flatname = basename(filename) + if flatname in flatnames: + flatnames.remove(flatname) + for filename, level, title in headings(filename): + toc.add(filename, level, title) + for flatname in flatnames: + if flatname in IGNORE_LIST: + continue + for filename, level, title in headings(f"docs/{flatname}"): + toc.add(filename, level, title) + toc.write("docs/README.md") + + +if __name__ == "__main__": + process() diff --git a/set/functions.go b/set/functions.go new file mode 100644 index 00000000..c2f4dd79 --- /dev/null +++ b/set/functions.go @@ -0,0 +1,99 @@ +package set + +import "sort" + +type ( + comparable interface { + string | int | int8 | int16 | int32 | int64 | uint | uint8 | uint16 | uint32 | uint64 | float32 | float64 + } +) + +func With[T comparable](incoming ...T) []T { + return Set(incoming) +} + +func Set[T comparable](incoming []T) []T { + return Keys(itemset(incoming)) +} + +func Values[Key, Value comparable](incoming map[Key]Value) []Value { + intermediate := make(map[Value]bool) + for _, value := range incoming { + intermediate[value] = true + } + return Keys(intermediate) +} + +func Keys[Key comparable, Value any](incoming map[Key]Value) []Key { + result := make([]Key, 0, len(incoming)) + for key, _ := range incoming { + result = append(result, key) + } + return Sort(result) +} + +func Sort[T comparable](set []T) []T { + sort.Slice(set, func(left, right int) bool { + return set[left] < set[right] + }) + return set +} + +func Member[T comparable](set []T, candidate T) bool { + for _, item := range set { + if candidate == item { + return true + } + } + return false +} + +func Membership[T comparable](set []T) map[T]bool { + result := make(map[T]bool) + for _, item := range set { + result[item] = true + } + return result +} + +func Update[T comparable](set []T, candidate T) ([]T, bool) { + if Member(set, candidate) { + return set, false + } + return Sort(append(set, candidate)), true +} + +func Intersect[T comparable](left, right []T) []T { + if len(right) < len(left) { + left, right = right, left + } + missing := len(left) + checked := itemset(left) + intermediate := make(map[T]bool) + for _, candidate := range right { + if checked[candidate] { + intermediate[candidate] = true + missing-- + } + if missing == 0 { + break + } + } + return Keys(intermediate) +} + +func Union[T comparable](left, right []T) []T { + intermediate := itemset(left) + for _, item := range right { + intermediate[item] = true + } + return Keys(intermediate) +} + +func itemset[T comparable](items []T) map[T]bool { + result := make(map[T]bool) + for _, item := range items { + result[item] = true + } + return result +} diff --git a/set/functions_test.go b/set/functions_test.go new file mode 100644 index 00000000..40ce1ba3 --- /dev/null +++ b/set/functions_test.go @@ -0,0 +1,44 @@ +package set_test + +import ( + "testing" + + "github.com/robocorp/rcc/hamlet" + "github.com/robocorp/rcc/set" +) + +func TestMakingAndAppending(t *testing.T) { + must_be, wont_be := hamlet.Specifications(t) + + original := set.Set([]string{"B", "A", "A", "B", "B", "A"}) + must_be.Text("[A B]", original) + + updated, ok := set.Update(original, "C") + must_be.True(ok) + must_be.Text("[A B C]", updated) + + already, ok := set.Update([]string{"A", "B", "C"}, "C") + wont_be.True(ok) + must_be.Text("[A B C]", already) +} + +func TestMembershipAndSorting(t *testing.T) { + must_be, wont_be := hamlet.Specifications(t) + + original := []string{"B", "A", "D", "F", "E", "C"} + must_be.True(set.Member(original, "F")) + wont_be.True(set.Member(original, "G")) + must_be.Text("[A B C D E F]", set.Sort(original)) + must_be.Text("[A B C D E F]", original) +} + +func TestOperations(t *testing.T) { + must_be, wont_be := hamlet.Specifications(t) + + smaller := set.With("D", "E", "F") + bigger := set.With("F", "A", "C", "E", "C", "A", "P") + must_be.True(set.Member(smaller, "D")) + wont_be.True(set.Member(bigger, "D")) + must_be.Text("[E F]", set.Intersect(smaller, bigger)) + must_be.Text("[A C D E F P]", set.Union(smaller, bigger)) +} diff --git a/settings/api.go b/settings/api.go new file mode 100644 index 00000000..1d0914ed --- /dev/null +++ b/settings/api.go @@ -0,0 +1,40 @@ +package settings + +import ( + "net/http" + + "github.com/robocorp/rcc/common" +) + +type EndpointsApi func(string) string + +type Api interface { + Name() string + Description() string + TemplatesYamlURL() string + Diagnostics(target *common.DiagnosticStatus) + Endpoint(string) string + Option(string) bool + DefaultEndpoint() string + IssuesURL() string + TelemetryURL() string + PypiURL() string + PypiTrustedHost() string + CondaURL() string + DownloadsLink(resource string) string + DocsLink(page string) string + PypiLink(page string) string + CondaLink(page string) string + Hostnames() []string + ConfiguredHttpTransport() *http.Transport + NoProxy() string + HttpsProxy() string + HttpProxy() string + HasPipRc() bool + HasMicroMambaRc() bool + HasCaBundle() bool + VerifySsl() bool + NoRevocation() bool + LegacyRenegotiation() bool + NoBuid() bool +} diff --git a/settings/data.go b/settings/data.go new file mode 100644 index 00000000..4374e4ca --- /dev/null +++ b/settings/data.go @@ -0,0 +1,315 @@ +package settings + +import ( + "encoding/json" + "net/url" + "sort" + "strings" + + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/pathlib" + "gopkg.in/yaml.v2" +) + +const ( + httpsPrefix = `https://` +) + +type StringMap map[string]string +type BoolMap map[string]bool + +func (it StringMap) Lookup(key string) string { + return it[key] +} + +// layer 0 is defaults from assets +// layer 1 is settings.yaml from disk +// layer 2 is "temporary" update layer +type SettingsLayers [3]*Settings + +func (it SettingsLayers) Effective() *Settings { + result := &Settings{ + Autoupdates: make(StringMap), + Branding: make(StringMap), + Certificates: &Certificates{}, + Network: &Network{}, + Endpoints: make(StringMap), + Options: make(BoolMap), + Hosts: make([]string, 0, 100), + Meta: &Meta{ + Name: "generated", + Description: "generated", + Source: "generated", + Version: "unknown", + }, + } + for _, layer := range it { + if layer != nil { + layer.onTopOf(result) + } + } + return result +} + +type Settings struct { + Autoupdates StringMap `yaml:"autoupdates,omitempty" json:"autoupdates,omitempty"` + Branding StringMap `yaml:"branding,omitempty" json:"branding,omitempty"` + Certificates *Certificates `yaml:"certificates,omitempty" json:"certificates,omitempty"` + Network *Network `yaml:"network,omitempty" json:"network,omitempty"` + Endpoints StringMap `yaml:"endpoints,omitempty" json:"endpoints,omitempty"` + Hosts []string `yaml:"diagnostics-hosts,omitempty" json:"diagnostics-hosts,omitempty"` + Options BoolMap `yaml:"options,omitempty" json:"options,omitempty"` + Meta *Meta `yaml:"meta,omitempty" json:"meta,omitempty"` +} + +func Empty() *Settings { + return &Settings{ + Meta: &Meta{ + Name: "generated", + Description: "generated", + Source: "generated", + Version: "unknown", + }, + } +} + +func FromBytes(raw []byte) (*Settings, error) { + var settings Settings + err := yaml.Unmarshal(raw, &settings) + if err != nil { + return nil, err + } + return &settings, nil +} + +func (it *Settings) onTopOf(target *Settings) { + for key, value := range it.Autoupdates { + if len(value) > 0 { + target.Autoupdates[key] = value + } + } + for key, value := range it.Branding { + if len(value) > 0 { + target.Branding[key] = value + } + } + for key, value := range it.Endpoints { + if len(value) > 0 { + target.Endpoints[key] = value + } + } + for key, value := range it.Options { + target.Options[key] = value + } + for _, host := range it.Hosts { + target.Hosts = append(target.Hosts, host) + } + if it.Certificates != nil { + it.Certificates.onTopOf(target) + } + if it.Network != nil { + it.Network.onTopOf(target) + } + if it.Meta != nil { + it.Meta.onTopOf(target) + } +} + +func (it *Settings) AsYaml() ([]byte, error) { + content, err := yaml.Marshal(it) + if err != nil { + return nil, err + } + return content, nil +} + +func (it *Settings) Source(filename string) *Settings { + if it.Meta != nil && len(filename) > 0 { + it.Meta.Source = filename + } + return it +} + +func (it *Settings) Hostnames() []string { + collector := make(map[string]bool) + if it.Endpoints != nil { + for _, name := range it.Endpoints { + hostFromUrl(name, collector) + } + } + if it.Hosts != nil { + for _, name := range it.Hosts { + collector[name] = true + } + } + result := make([]string, 0, len(collector)) + for key, _ := range collector { + result = append(result, key) + } + sort.Strings(result) + return result +} + +func (it *Settings) AsJson() ([]byte, error) { + content, err := json.MarshalIndent(it, "", " ") + if err != nil { + return nil, err + } + return content, nil +} + +func diagnoseUrl(link, label string, diagnose common.Diagnoser, correct bool) bool { + if len(link) == 0 { + diagnose.Fatal(0, "", "required %q URL is missing.", label) + return false + } + if !strings.HasPrefix(link, httpsPrefix) { + diagnose.Fatal(0, "", "%q URL %q is does not start with %q prefix.", label, link, httpsPrefix) + return false + } + _, err := url.Parse(link) + if err != nil { + diagnose.Fatal(0, "", "%q URL %q cannot be parsed, reason %v.", label, link, err) + return false + } + return correct +} + +func diagnoseOptionalUrl(link, label string, diagnose common.Diagnoser, correct bool) bool { + if len(strings.TrimSpace(link)) == 0 { + return correct + } else { + return diagnoseUrl(link, label, diagnose, correct) + } +} + +func (it *Settings) CriticalEnvironmentDiagnostics(target *common.DiagnosticStatus) { + diagnose := target.Diagnose("settings.yaml") + correct := true + if it.Endpoints == nil { + diagnose.Fatal(0, "", "endpoints section is totally missing") + correct = false + } else { + correct = diagnoseUrl(it.Endpoints["cloud-api"], "endpoints/cloud-api", diagnose, correct) + correct = diagnoseUrl(it.Endpoints["downloads"], "endpoints/downloads", diagnose, correct) + } + if correct { + diagnose.Ok(0, "Critical environment diagnostics are ok.") + } +} + +func (it *Settings) Diagnostics(target *common.DiagnosticStatus) { + diagnose := target.Diagnose("Settings") + correct := true + if it.Certificates == nil { + diagnose.Warning(0, "", "settings.yaml: certificates section is totally missing") + correct = false + } + if it.Endpoints == nil { + diagnose.Warning(0, "", "settings.yaml: endpoints section is totally missing") + correct = false + } else { + correct = diagnoseUrl(it.Endpoints["cloud-api"], "endpoints/cloud-api", diagnose, correct) + correct = diagnoseUrl(it.Endpoints["downloads"], "endpoints/downloads", diagnose, correct) + + correct = diagnoseOptionalUrl(it.Endpoints["cloud-ui"], "endpoints/cloud-ui", diagnose, correct) + correct = diagnoseOptionalUrl(it.Endpoints["cloud-linking"], "endpoints/cloud-linking", diagnose, correct) + correct = diagnoseOptionalUrl(it.Endpoints["issues"], "endpoints/issues", diagnose, correct) + correct = diagnoseOptionalUrl(it.Endpoints["telemetry"], "endpoints/telemetry", diagnose, correct) + correct = diagnoseOptionalUrl(it.Endpoints["docs"], "endpoints/docs", diagnose, correct) + correct = diagnoseOptionalUrl(it.Endpoints["conda"], "endpoints/conda", diagnose, correct) + correct = diagnoseOptionalUrl(it.Endpoints["pypi"], "endpoints/pypi", diagnose, correct) + correct = diagnoseOptionalUrl(it.Endpoints["pypi-trusted"], "endpoints/pypi-trusted", diagnose, correct) + } + if it.Meta == nil { + diagnose.Warning(0, "", "settings.yaml: meta section is totally missing") + correct = false + } + if correct { + diagnose.Ok(0, "In general, 'settings.yaml' is ok.") + } +} + +type Certificates struct { + VerifySsl bool `yaml:"verify-ssl" json:"verify-ssl"` + SslNoRevoke bool `yaml:"ssl-no-revoke" json:"ssl-no-revoke"` + LegacyRenegotiation bool `yaml:"legacy-renegotiation-allowed" json:"legacy-renegotiation-allowed"` + CaBundle string `yaml:"ca-bundle,omitempty" json:"ca-bundle,omitempty"` +} + +func (it *Certificates) onTopOf(target *Settings) { + if target.Certificates == nil { + target.Certificates = &Certificates{} + } + target.Certificates.VerifySsl = it.VerifySsl + target.Certificates.SslNoRevoke = it.SslNoRevoke + target.Certificates.LegacyRenegotiation = it.LegacyRenegotiation + if pathlib.IsFile(common.CaBundleFile()) { + target.Certificates.CaBundle = common.CaBundleFile() + } +} + +func justHostAndPort(link string) string { + if len(link) == 0 { + return "" + } + parsed, err := url.Parse(link) + if err != nil { + return "" + } + return parsed.Host +} + +func hostFromUrl(link string, collector map[string]bool) { + host := justHostAndPort(link) + if len(host) > 0 { + parts := strings.SplitN(host, ":", 2) + collector[parts[0]] = true + } +} + +type Meta struct { + Name string `yaml:"name" json:"name"` + Description string `yaml:"description" json:"description"` + Source string `yaml:"source" json:"source"` + Version string `yaml:"version" json:"version"` +} + +func (it *Meta) onTopOf(target *Settings) { + if target.Meta == nil { + target.Meta = &Meta{} + } + if len(it.Name) > 0 { + target.Meta.Name = it.Name + } + if len(it.Description) > 0 { + target.Meta.Description = it.Description + } + if len(it.Source) > 0 { + target.Meta.Source = it.Source + } + if len(it.Version) > 0 { + target.Meta.Version = it.Version + } +} + +type Network struct { + NoProxy string `yaml:"no-proxy" json:"no-proxy"` + HttpsProxy string `yaml:"https-proxy" json:"https-proxy"` + HttpProxy string `yaml:"http-proxy" json:"http-proxy"` +} + +func (it *Network) onTopOf(target *Settings) { + if target.Network == nil { + target.Network = &Network{} + } + if len(it.NoProxy) > 0 { + target.Network.NoProxy = it.NoProxy + } + if len(it.HttpsProxy) > 0 { + target.Network.HttpsProxy = it.HttpsProxy + } + if len(it.HttpProxy) > 0 { + target.Network.HttpProxy = it.HttpProxy + } +} diff --git a/settings/platform.go b/settings/platform.go new file mode 100644 index 00000000..c6b0e5f7 --- /dev/null +++ b/settings/platform.go @@ -0,0 +1,41 @@ +package settings + +import ( + "regexp" + "strings" + + "github.com/robocorp/rcc/shell" +) + +const ( + liningPattern = `\r?\n` + spacingPattern = `\s+` +) + +var ( + spacingForm = regexp.MustCompile(spacingPattern) + liningForm = regexp.MustCompile(liningPattern) +) + +func operatingSystem() string { + output, _, err := shell.New(nil, ".", osInfoCommand...).NoStderr().CaptureOutput() + if err != nil { + output = err.Error() + } + return output +} + +func pickLines(text string) []string { + result := []string{} + for _, part := range liningForm.Split(text, -1) { + flat := strings.TrimSpace(strings.Join(spacingForm.Split(part, -1), " ")) + if len(flat) > 0 { + result = append(result, flat) + } + } + return result +} + +func OperatingSystem() string { + return strings.Join(pickLines(operatingSystem()), "; ") +} diff --git a/settings/platform_darwin.go b/settings/platform_darwin.go new file mode 100644 index 00000000..4d51b2ac --- /dev/null +++ b/settings/platform_darwin.go @@ -0,0 +1,5 @@ +package settings + +var ( + osInfoCommand = []string{"sw_vers"} +) diff --git a/settings/platform_linux.go b/settings/platform_linux.go new file mode 100644 index 00000000..c3830db4 --- /dev/null +++ b/settings/platform_linux.go @@ -0,0 +1,5 @@ +package settings + +var ( + osInfoCommand = []string{"lsb_release", "-a"} +) diff --git a/settings/platform_windows.go b/settings/platform_windows.go new file mode 100644 index 00000000..04c967c3 --- /dev/null +++ b/settings/platform_windows.go @@ -0,0 +1,5 @@ +package settings + +var ( + osInfoCommand = []string{"cmd.exe", "/c", "ver"} +) diff --git a/settings/profile.go b/settings/profile.go new file mode 100644 index 00000000..c05a6278 --- /dev/null +++ b/settings/profile.go @@ -0,0 +1,100 @@ +package settings + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/fail" + "github.com/robocorp/rcc/pathlib" + "gopkg.in/yaml.v2" +) + +type Profile struct { + Name string `yaml:"name" json:"name"` + Description string `yaml:"description" json:"description"` + Settings *Settings `yaml:"settings,omitempty" json:"settings,omitempty"` + PipRc string `yaml:"piprc,omitempty" json:"piprc,omitempty"` + MicroMambaRc string `yaml:"micromambarc,omitempty" json:"micromambarc,omitempty"` + CaBundle string `yaml:"ca-bundle,omitempty" json:"ca-bundle,omitempty"` +} + +func (it *Profile) AsYaml() ([]byte, error) { + content, err := yaml.Marshal(it) + if err != nil { + return nil, err + } + return content, nil +} + +func (it *Profile) SaveAs(filename string) error { + body, err := it.AsYaml() + if err != nil { + return err + } + return pathlib.WriteFile(filename, body, 0o666) +} + +func (it *Profile) LoadFrom(filename string) error { + raw, err := os.ReadFile(filename) + if err != nil { + return err + } + return yaml.Unmarshal(raw, it) +} + +func (it *Profile) Import() (err error) { + basename := fmt.Sprintf("profile_%s.yaml", strings.ToLower(it.Name)) + filename := common.ExpandPath(filepath.Join(common.Product.Home(), basename)) + return it.SaveAs(filename) +} + +func (it *Profile) Activate() (err error) { + defer fail.Around(&err) + + err = it.Remove() + fail.On(err != nil, "%s", err) + if it.Settings != nil { + body, err := it.Settings.AsYaml() + fail.On(err != nil, "Failed to parse settings.yaml, reason: %v", err) + err = saveIfBody(common.SettingsFile(), body) + fail.On(err != nil, "Failed to save settings.yaml, reason: %v", err) + } + err = saveIfBody(common.PipRcFile(), []byte(it.PipRc)) + fail.On(err != nil, "Failed to save piprc, reason: %v", err) + err = saveIfBody(common.MicroMambaRcFile(), []byte(it.MicroMambaRc)) + fail.On(err != nil, "Failed to save micromambarc, reason: %v", err) + err = saveIfBody(common.CaBundleFile(), []byte(it.CaBundle)) + fail.On(err != nil, "Failed to save ca-bundle.pem, reason: %v", err) + return nil +} + +func (it *Profile) Remove() (err error) { + defer fail.Around(&err) + + err = removeIfExists(common.PipRcFile()) + fail.On(err != nil, "Failed to remove piprc, reason: %v", err) + err = removeIfExists(common.MicroMambaRcFile()) + fail.On(err != nil, "Failed to remove micromambarc, reason: %v", err) + err = removeIfExists(common.SettingsFile()) + fail.On(err != nil, "Failed to remove settings.yaml, reason: %v", err) + err = removeIfExists(common.CaBundleFile()) + fail.On(err != nil, "Failed to remove ca-bundle.pem, reason: %v", err) + return nil +} + +func removeIfExists(filename string) error { + if !pathlib.Exists(filename) { + return nil + } + return os.Remove(filename) +} + +func saveIfBody(filename string, body []byte) error { + if body != nil && len(body) > 0 { + return pathlib.WriteFile(filename, body, 0o666) + } + return nil +} diff --git a/settings/settings.go b/settings/settings.go new file mode 100644 index 00000000..26a3ddf5 --- /dev/null +++ b/settings/settings.go @@ -0,0 +1,316 @@ +package settings + +import ( + "crypto/tls" + "crypto/x509" + "fmt" + "io" + "net/http" + "net/url" + "os" + + "github.com/robocorp/rcc/blobs" + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/pathlib" + "github.com/robocorp/rcc/pretty" +) + +const ( + pypiDefault = "https://pypi.org/simple/" + condaDefault = "https://conda.anaconda.org/" +) + +var ( + httpTransport *http.Transport + cachedSettings *Settings + Global gateway + chain SettingsLayers +) + +func cacheSettings(result *Settings) (*Settings, error) { + if result != nil { + cachedSettings = result + } + return result, nil +} + +func HasCustomSettings() bool { + return pathlib.IsFile(common.SettingsFile()) +} + +func DefaultSettings() ([]byte, error) { + return blobs.Asset(common.Product.DefaultSettingsYamlFile()) +} + +func DefaultSettingsLayer() *Settings { + content, err := DefaultSettings() + pretty.Guard(err == nil, 111, "Could not read default settings, reason: %v", err) + config, err := FromBytes(content) + pretty.Guard(err == nil, 111, "Could not parse default settings, reason: %v", err) + return config +} + +func CustomSettingsLayer() *Settings { + if !HasCustomSettings() { + return nil + } + config, err := LoadSetting(common.SettingsFile()) + pretty.Guard(err == nil, 111, "Could not load/parse custom settings, reason: %v", err) + return config +} + +func LoadSetting(filename string) (*Settings, error) { + content, err := os.ReadFile(filename) + if err != nil { + return nil, err + } + config, err := FromBytes(content) + if err != nil { + return nil, err + } + return config, nil +} + +func SummonSettings() (*Settings, error) { + if cachedSettings != nil { + return cachedSettings, nil + } + return cacheSettings(chain.Effective()) +} + +func showDiagnosticsChecks(sink io.Writer, details *common.DiagnosticStatus) { + fmt.Fprintln(sink, "Checks:") + for _, check := range details.Checks { + fmt.Fprintf(sink, " - %-8s %-8s %s\n", check.Type, check.Status, check.Message) + } +} + +func CriticalEnvironmentSettingsCheck() { + config, err := SummonSettings() + pretty.Guard(err == nil, 80, "Aborting! Could not even get setting, reason: %v", err) + result := &common.DiagnosticStatus{ + Details: make(map[string]string), + Checks: []*common.DiagnosticCheck{}, + } + config.CriticalEnvironmentDiagnostics(result) + diagnose := result.Diagnose("Settings") + if HasCustomSettings() { + diagnose.Ok(0, "Uses custom settings at %q.", common.SettingsFile()) + } else { + diagnose.Ok(0, "Uses builtin settings.") + } + fatal, fail, _, _ := result.Counts() + if (fatal + fail) > 0 { + showDiagnosticsChecks(os.Stderr, result) + pretty.Guard(false, 111, "\nBroken settings.yaml. Cannot continue!") + } +} + +func resolveLink(link, page string) string { + docs, err := url.Parse(link) + if err != nil { + return page + } + local, err := url.Parse(page) + if err != nil { + return page + } + return docs.ResolveReference(local).String() +} + +type gateway bool + +func (it gateway) settings() *Settings { + config, err := SummonSettings() + pretty.Guard(err == nil, 111, "Could not get settings, reason: %v", err) + return config +} + +func (it gateway) Name() string { + return it.settings().Meta.Name +} + +func (it gateway) Description() string { + return it.settings().Meta.Description +} + +func (it gateway) TemplatesYamlURL() string { + return it.settings().Autoupdates["templates"] +} + +func (it gateway) Diagnostics(target *common.DiagnosticStatus) { + it.settings().Diagnostics(target) +} + +func (it gateway) Endpoint(key string) string { + return it.settings().Endpoints[key] +} + +func (it gateway) Option(key string) bool { + value, ok := it.settings().Options[key] + return ok && value +} + +func (it gateway) DefaultEndpoint() string { + return it.Endpoint("cloud-api") +} + +func (it gateway) IssuesURL() string { + return it.Endpoint("issues") +} + +func (it gateway) TelemetryURL() string { + return it.Endpoint("telemetry") +} + +func (it gateway) PypiURL() string { + return it.Endpoint("pypi") +} + +func (it gateway) PypiTrustedHost() string { + return justHostAndPort(it.Endpoint("pypi-trusted")) +} + +func (it gateway) CondaURL() string { + return it.Endpoint("conda") +} + +func (it gateway) NoProxy() string { + return it.settings().Network.NoProxy +} + +func (it gateway) HttpsProxy() string { + return it.settings().Network.HttpsProxy +} + +func (it gateway) HttpProxy() string { + return it.settings().Network.HttpProxy +} +func (it gateway) HasPipRc() bool { + return pathlib.IsFile(common.PipRcFile()) +} + +func (it gateway) HasMicroMambaRc() bool { + return pathlib.IsFile(common.MicroMambaRcFile()) +} + +func (it gateway) HasCaBundle() bool { + return pathlib.IsFile(common.CaBundleFile()) +} + +func (it gateway) DownloadsLink(resource string) string { + return resolveLink(it.Endpoint("downloads"), resource) +} + +func (it gateway) DocsLink(page string) string { + return resolveLink(it.Endpoint("docs"), page) +} + +func (it gateway) PypiLink(page string) string { + endpoint := it.Endpoint("pypi") + if len(endpoint) == 0 { + endpoint = pypiDefault + } + return resolveLink(endpoint, page) +} + +func (it gateway) CondaLink(page string) string { + endpoint := it.Endpoint("conda") + if len(endpoint) == 0 { + endpoint = condaDefault + } + return resolveLink(endpoint, page) +} + +func (it gateway) Hostnames() []string { + return it.settings().Hostnames() +} +func (it gateway) VerifySsl() bool { + return it.settings().Certificates.VerifySsl +} + +func (it gateway) LegacyRenegotiation() bool { + return it.settings().Certificates.LegacyRenegotiation +} + +func (it gateway) NoRevocation() bool { + return it.settings().Certificates.SslNoRevoke +} + +func (it gateway) NoBuild() bool { + nobuild := len(os.Getenv("RCC_NO_BUILD")) > 0 + return nobuild || common.NoBuild || it.Option("no-build") +} + +func (it gateway) ConfiguredHttpTransport() *http.Transport { + return httpTransport.Clone() +} + +func (it gateway) loadRootCAs() *x509.CertPool { + roots, err := x509.SystemCertPool() + if err != nil { + roots = x509.NewCertPool() + } + + if !it.HasCaBundle() { + return roots + } + + common.Debug("Using CA bundle file from %q.", common.CaBundleFile()) + certificates, err := os.ReadFile(common.CaBundleFile()) + if err != nil { + common.Log("Warning! Problem reading %q, reason: %v", common.CaBundleFile(), err) + return roots + } + + ok := roots.AppendCertsFromPEM(certificates) + if !ok { + common.Log("Warning! Problem appending certificates from %q.", common.CaBundleFile()) + } + return roots +} + +func initProtection() { + status := recover() + if status != nil { + fmt.Fprintf(os.Stderr, "Fatal failure with '%v', see: %v\n", common.SettingsFile(), status) + fmt.Fprintln(os.Stderr, "Sorry. Cannot recover, will exit now!") + os.Exit(111) + } +} + +func init() { + defer initProtection() + + chain = SettingsLayers{ + DefaultSettingsLayer(), + CustomSettingsLayer(), + nil, + } + verifySsl := true + Global = gateway(true) + httpTransport = http.DefaultTransport.(*http.Transport).Clone() + settings, err := SummonSettings() + if err == nil && settings.Certificates != nil { + verifySsl = settings.Certificates.VerifySsl + } + proxyUrl := "" + if len(Global.HttpProxy()) > 0 { + proxyUrl = Global.HttpProxy() + } + if len(Global.HttpsProxy()) > 0 { + proxyUrl = Global.HttpsProxy() + } + if len(proxyUrl) > 0 { + link, err := url.Parse(proxyUrl) + if err != nil { + common.Log("Warning! Problem parsing proxy URL %q, reason: %v.", proxyUrl, err) + } else { + httpTransport.Proxy = http.ProxyURL(link) + } + } + httpTransport.TLSClientConfig = &tls.Config{ + InsecureSkipVerify: !verifySsl, + RootCAs: Global.loadRootCAs(), + } +} diff --git a/settings/settings_test.go b/settings/settings_test.go new file mode 100644 index 00000000..b2da0ed1 --- /dev/null +++ b/settings/settings_test.go @@ -0,0 +1,40 @@ +package settings_test + +import ( + "testing" + + "github.com/robocorp/rcc/hamlet" + "github.com/robocorp/rcc/settings" +) + +func TestCanCallEntropyFunction(t *testing.T) { + must_be, wont_be := hamlet.Specifications(t) + + sut, err := settings.SummonSettings() + must_be.Nil(err) + wont_be.Nil(sut) + + wont_be.Nil(settings.Global) + must_be.True(len(settings.Global.Hostnames()) > 1) + + must_be.Equal("https://robocorp.com/docs/hello.html", settings.Global.DocsLink("hello.html")) + must_be.Equal("https://robocorp.com/docs/products/manual.html", settings.Global.DocsLink("products/manual.html")) +} + +func TestThatSomeDefaultValuesAreVisible(t *testing.T) { + must_be, wont_be := hamlet.Specifications(t) + + sut, err := settings.SummonSettings() + must_be.Nil(err) + wont_be.Nil(sut) + + must_be.Equal("https://api.eu1.robocorp.com/", settings.Global.DefaultEndpoint()) + must_be.Equal("https://telemetry.robocorp.com/", settings.Global.TelemetryURL()) + must_be.Equal("", settings.Global.PypiURL()) + must_be.Equal("", settings.Global.PypiTrustedHost()) + must_be.Equal("", settings.Global.CondaURL()) + must_be.Equal("", settings.Global.HttpProxy()) + must_be.Equal("", settings.Global.HttpsProxy()) + must_be.Equal("", settings.Global.NoProxy()) + must_be.Equal(9, len(settings.Global.Hostnames())) +} diff --git a/shell/task.go b/shell/task.go index 00e08615..e5907390 100644 --- a/shell/task.go +++ b/shell/task.go @@ -1,17 +1,41 @@ package shell import ( + "bytes" "io" "os" "os/exec" + "os/signal" "path/filepath" + "time" + + "github.com/google/shlex" + "github.com/robocorp/rcc/common" + "github.com/robocorp/rcc/pathlib" + "github.com/robocorp/rcc/pretty" ) -type Task struct { - environment []string - directory string - executable string - args []string +type ( + Common interface { + Debug(string, ...interface{}) error + Trace(string, ...interface{}) error + Timeline(string, ...interface{}) + } + + Task struct { + environment []string + directory string + executable string + args []string + stderronly bool + nostderr bool + } + + Wrapper func() +) + +func Split(commandline string) ([]string, error) { + return shlex.Split(commandline) } func New(environment []string, directory string, task ...string) *Task { @@ -21,17 +45,55 @@ func New(environment []string, directory string, task ...string) *Task { directory: directory, executable: executable, args: args, + stderronly: false, + nostderr: false, } } +func (it *Task) StderrOnly() *Task { + it.stderronly = true + return it +} + +func (it *Task) NoStderr() *Task { + it.nostderr = true + return it +} + +func (it *Task) stdout() io.Writer { + if it.stderronly { + return os.Stderr + } + return os.Stdout +} + func (it *Task) execute(stdin io.Reader, stdout, stderr io.Writer) (int, error) { + common.Trace("Execute %q with arguments %q", it.executable, it.args) command := exec.Command(it.executable, it.args...) command.Env = it.environment command.Dir = it.directory command.Stdin = stdin command.Stdout = stdout - command.Stderr = stderr - err := command.Run() + if it.nostderr { + command.Stderr = nil + } else { + command.Stderr = stderr + } + command.WaitDelay = 3 * time.Second + err := command.Start() + if err != nil { + return -500, err + } + common.Timeline("exec %q started", it.executable) + common.Debug("PID #%d is %q.", command.Process.Pid, command) + defer func() { + if command.ProcessState.ExitCode() != 0 { + common.Log("Process %d: %v, command: %s %s [%s/%d]", command.Process.Pid, command.ProcessState, it.executable, it.args, common.Version, os.Getpid()) + } else { + common.Debug("PID #%d finished: %v.", command.Process.Pid, command.ProcessState) + } + }() + err = command.Wait() exit, ok := err.(*exec.ExitError) if ok { return exit.ExitCode(), err @@ -43,7 +105,15 @@ func (it *Task) execute(stdin io.Reader, stdout, stderr io.Writer) (int, error) } func (it *Task) Transparent() (int, error) { - return it.execute(os.Stdin, os.Stdout, os.Stderr) + return it.execute(os.Stdin, it.stdout(), os.Stderr) +} + +func (it *Task) Execute(interactive bool) (int, error) { + var stdin io.Reader = os.Stdin + if !interactive { + stdin = bytes.NewReader([]byte{}) + } + return it.execute(stdin, it.stdout(), os.Stderr) } func (it *Task) Tee(folder string, interactive bool) (int, error) { @@ -51,20 +121,61 @@ func (it *Task) Tee(folder string, interactive bool) (int, error) { if err != nil { return -600, err } - outfile, err := os.Create(filepath.Join(folder, "stdout.log")) + outfile, err := pathlib.Create(filepath.Join(folder, "stdout.log")) if err != nil { return -601, err } defer outfile.Close() - errfile, err := os.Create(filepath.Join(folder, "stderr.log")) + errfile, err := pathlib.Create(filepath.Join(folder, "stderr.log")) if err != nil { return -602, err } defer errfile.Close() - stdout := io.MultiWriter(os.Stdout, outfile) + stdout := io.MultiWriter(it.stdout(), outfile) stderr := io.MultiWriter(os.Stderr, errfile) + var stdin io.Reader = os.Stdin + if !interactive { + stdin = bytes.NewReader([]byte{}) + } + return it.execute(stdin, stdout, stderr) +} + +func (it *Task) Observed(sink io.Writer, interactive bool) (int, error) { + stdout := io.MultiWriter(it.stdout(), sink) + stderr := io.MultiWriter(os.Stderr, sink) + var stdin io.Reader = os.Stdin if !interactive { - os.Stdin.Close() + stdin = bytes.NewReader([]byte{}) } - return it.execute(os.Stdin, stdout, stderr) + return it.execute(stdin, stdout, stderr) +} + +func (it *Task) Tracked(sink io.Writer, interactive bool) (int, error) { + var stdin io.Reader = os.Stdin + if !interactive { + stdin = bytes.NewReader([]byte{}) + } + return it.execute(stdin, sink, sink) +} + +func (it *Task) CaptureOutput() (string, int, error) { + stdin := bytes.NewReader([]byte{}) + stdout := bytes.NewBuffer(nil) + code, err := it.execute(stdin, stdout, os.Stderr) + return stdout.String(), code, err +} + +func WithInterrupt(task Wrapper) { + signals := make(chan os.Signal, 1) + defer signal.Stop(signals) + defer close(signals) + go func() { + signal.Notify(signals, os.Interrupt) + got, ok := <-signals + if ok { + pretty.Note("Detected and ignored %q signal. Second one will not be ignored. [rcc]", got) + } + signal.Stop(signals) + }() + task() } diff --git a/shell/transparent_test.go b/shell/transparent_test.go index 4f04a6af..92466771 100644 --- a/shell/transparent_test.go +++ b/shell/transparent_test.go @@ -25,5 +25,5 @@ func TestCanExecuteSimpleEcho(t *testing.T) { code, err = shell.New(nil, ".", "ls", "-l", "crapiti.crap").Transparent() wont_be.Nil(err) - must_be.Equal(2, code) + wont_be.Equal(0, code) } diff --git a/tasks.py b/tasks.py new file mode 100644 index 00000000..0d41d1c4 --- /dev/null +++ b/tasks.py @@ -0,0 +1,256 @@ +import os +import shutil +import sys + +from invoke import task + +# Determine OS-specific commands +if sys.platform == "win32": + PYTHON = "python" + LS = "dir" + WHICH = "where" +else: + PYTHON = "python3" + LS = "ls -l" + WHICH = "which -a" + + +@task +def what(c): + """Show latest HEAD with stats""" + c.run("go version") + c.run("git --no-pager log -2 --stat HEAD") + + +@task +def tooling(c): + """Display tooling information""" + print(f"PATH is {os.environ['PATH']}") + print(f"GOPATH is {os.environ.get('GOPATH', 'Not set')}") + print(f"GOROOT is {os.environ.get('GOROOT', 'Not set')}") + print("git info:") + c.run(f"{WHICH} git || echo NA") + + +@task +def noassets(c): + """Remove asset files""" + import glob + + patterns = [ + "blobs/assets/micromamba.*", + "blobs/assets/*.zip", + "blobs/assets/*.yaml", + "blobs/assets/*.py", + "blobs/assets/man/*.txt", + "blobs/docs/*.md", + ] + for pattern in patterns: + for file_path in glob.glob(pattern, recursive=True): + try: + os.remove(file_path) + print(f"Removed: {file_path}") + except OSError as e: + print(f"Error removing {file_path}: {e}") + + +def download_link(version, platform, filename): + return f"https://downloads.robocorp.com/micromamba/{version}/{platform}/{filename}" + + +@task +def micromamba(c): + """Download micromamba files""" + with open("assets/micromamba_version.txt", "r", encoding="utf-8") as f: + version = f.read().strip() + print(f"Using micromamba version {version}") + + platforms = { + "macos64": "darwin_amd64", + "windows64": "windows_amd64", + "linux64": "linux_amd64", + } + + for platform, arch in platforms.items(): + filename = "micromamba.exe" if platform == "windows64" else "micromamba" + url = download_link(version, platform, filename) + output = f"blobs/assets/micromamba.{arch}" + if os.path.exists(output + ".gz"): + print(f"Asset {output}.gz already exists, skipping") + continue + print(f"Downloading {url} to {output}") + c.run(f"curl -o {output} {url}") + print(f"Compressing {output}") + c.run(f"gzip -f -9 {output}") + + +@task(pre=[micromamba]) +def assets(c): + """Prepare asset files""" + import glob + from zipfile import ZIP_DEFLATED, ZipFile + + # Process template directories + for directory in glob.glob("templates/*/"): + basename = os.path.basename(os.path.dirname(directory)) + assetname = os.path.abspath(f"blobs/assets/{basename}.zip") + + if os.path.exists(assetname): + print(f"Asset {assetname} already exists, skipping") + continue + + print(f"Directory {directory} => {assetname}") + + with ZipFile(assetname, "w", ZIP_DEFLATED) as zipf: + for root, _, files in os.walk(directory): + for file in files: + file_path = os.path.join(root, file) + arcname = os.path.relpath(file_path, directory) + zipf.write(file_path, arcname) + + # Copy asset files + asset_patterns = ["assets/*.txt", "assets/*.yaml", "assets/*.py"] + for pattern in asset_patterns: + for file in glob.glob(pattern): + print(f"Copying {file} to blobs/assets/") + shutil.copy(file, "blobs/assets/") + + # Copy man pages + os.makedirs("blobs/assets/man", exist_ok=True) + for file in glob.glob("assets/man/*.txt"): + print(f"Copying {file} to blobs/assets/man/") + shutil.copy(file, "blobs/assets/man/") + + # Copy docs + os.makedirs("blobs/docs", exist_ok=True) + for file in glob.glob("docs/*.md"): + print(f"Copying {file} to blobs/docs/") + shutil.copy(file, "blobs/docs/") + + +@task(pre=[noassets]) +def clean(c): + """Remove build directory""" + shutil.rmtree("build", ignore_errors=True) + print("Removed build directory") + + +@task +def toc(c): + """Update table of contents on docs/ directory""" + c.run(f"{PYTHON} scripts/toc.py") + print("Ran scripts/toc.py") + + +@task(pre=[toc]) +def support(c): + """Create necessary directories""" + for dir in ["tmp", "build/linux64", "build/macos64", "build/windows64"]: + os.makedirs(dir, exist_ok=True) + + +@task(pre=[support, assets]) +def test(c, cover=False): + """Run tests""" + os.environ["GOARCH"] = "amd64" + if cover: + c.run("go test -cover -coverprofile=tmp/cover.out ./...") + c.run("go tool cover -func=tmp/cover.out") + else: + c.run("go test ./...") + + +def version() -> str: + import re + + with open("common/version.go", "r") as file: + content = file.read() + match = re.search(r"Version\s*=\s*`v([^`]+)`", content) + if match: + return match.group(1) + else: + raise ValueError("Version not found in common/version.go") + + +@task +def version_txt(c): + """Create version.txt file""" + support(c) + target = "build/version.txt" + v = version() + with open(target, "w") as f: + f.write(f"v{v}") + print(f"Created {target} with version {v}") + + +@task(pre=[support, version_txt, assets]) +def build(c, platform="all"): + """Build executables""" + from pathlib import Path + + os.environ["CGO_ENABLED"] = "0" + os.environ["GOARCH"] = "amd64" + + build_platforms = ["linux", "darwin", "windows"] + + if platform == "all": + platforms = build_platforms + else: + assert platform in build_platforms, f"Invalid platform: {platform}" + platforms = [platform] + + for goos in platforms: + os.environ["GOOS"] = goos + output = f"build/{goos}64/" + + c.run(f"go build -ldflags -s -o {output} ./cmd/...") + + ext = ".exe" if goos == "windows" else "" + f = f"{output}rcc{ext}" + assert Path(f).exists(), f"File {f} does not exist" + print(f"Built: {f}") + + +@task +def windows64(c): + """Build windows64 executable""" + build(c, platform="windows") + + +@task +def linux64(c): + """Build linux64 executable""" + build(c, platform="linux") + + +@task +def macos64(c): + """Build macos64 executable""" + build(c, platform="darwin") + + +@task +def robotsetup(c): + """Setup build environment""" + if not os.path.exists("robot_requirements.txt"): + raise RuntimeError( + f"robot_requirements.txt not found. Current directory: {os.path.abspath(os.getcwd())}" + ) + c.run(f"{PYTHON} -m pip install --upgrade -r robot_requirements.txt") + c.run(f"{PYTHON} -m pip freeze") + + +@task +def local(c, do_test=True): + """Build local, operating system specific rcc""" + tooling(c) + if do_test: + test(c) + c.run("go build -o build/ ./cmd/...") + + +@task(pre=[robotsetup, assets, local]) +def robot(c): + """Run robot tests on local application""" + print("Running robot tests...") + c.run(f"{PYTHON} -m robot -L DEBUG -d tmp/output robot_tests") diff --git a/templates/extended/.gitignore b/templates/extended/.gitignore old mode 100644 new mode 100755 diff --git a/templates/extended/LICENSE b/templates/extended/LICENSE deleted file mode 100644 index 9c312761..00000000 --- a/templates/extended/LICENSE +++ /dev/null @@ -1 +0,0 @@ - diff --git a/templates/extended/README.md b/templates/extended/README.md index f1e07425..262c1952 100644 --- a/templates/extended/README.md +++ b/templates/extended/README.md @@ -1,3 +1,64 @@ -# README for this robot +# README for the robot - +Describe your robot here. +E.g. what it does, what are the requirements, how to run it. + + +## Development guide + +Run the robot locally: +``` +rcc run +``` + +Provide access credentials for Robocorp Control Room connectivity: +``` +rcc configure credentials +``` + +Upload to Robocorp Control Room: +``` +rcc cloud push --workspace --robot +``` + +### Suggested directory structure + +The directory structure given by the template: +``` +├── devdata +├── keywords +│ └── keywords.robot +├── libraries +│ └── Library.py +├── variables +│ └── variables.py +├── conda.yaml +├── robot.yaml +└── tasks.robot +``` + +where +* `devdata`: place for all data/material related to development, e.g. test data. Do not put any sensitive data here! +* `keywords`: Robot Framework keyword files. +* `libraries`: Python library code. +* `variables`: Define your robot variables in a centralized place. Do not put any sensitive data here! +* `conda.yaml`: Environment configuration file. +* `robot.yaml`: Robot configuration file. +* `tasks.robot`: Robot Framework task suite - high level process definition. + +In addition to these you can create your own directories (e.g. `bin`, `tmp`). Add these directories to the `PATH` or `PYTHONPATH` section of `robot.yaml` if necessary. + +Logs and artifacts are stored in `output` by default - see `robot.yaml` for configuring this. + +See [Docs](https://robocorp.com/docs/development-howtos/variables-and-secrets/) for handling variables and secrets. + + +### Configuration + +Give the task name and startup commands in `robot.yaml` with some additional configuration. See [Docs](https://robocorp.com/docs/setup/robot-structure#robot-configuration-file-robot-yaml) for more. + + +Put all the robot dependencies in `conda.yaml`. Robocorp tools (and rcc) uses [Conda](https://docs.conda.io) for managing the execution environment. For development you can also install packages manually with `pip`. + +### Additional documentation +See [Robocorp Docs](https://robocorp.com/docs/) for more documentation. diff --git a/templates/extended/TODO.md b/templates/extended/TODO.md deleted file mode 100644 index 722a1f8a..00000000 --- a/templates/extended/TODO.md +++ /dev/null @@ -1,38 +0,0 @@ -# TODO file for a new robot - -- Read all the `INTRODUCTION.md` files and follow their guidance (the files - should help you get an understanding of what belongs where in this new - robot). -- Add your content to the top-level `README.md` file. -- Add your license text to the `LICENSE` file. -- Make your changes to this project and make it yours. -- Refer to the "Full workflow with CLI" below for running your task. - -# Full workflow with CLI - -In the shell (or the command prompt), do the following: - -## Creating a new task - -```bash -rcc robot init --directory new_robot -cd new_robot -``` - -## Running the task in place - -``` -rcc task run --robot robot.yaml -``` - -## Providing access credentials for Robocorp Cloud connectivity - -```bash -rcc configure credentials -``` - -## Uploading to Robocorp Cloud - -```bash -rcc cloud push --workspace 111 --robot 111 -``` diff --git a/templates/extended/bin/INTRODUCTION.md b/templates/extended/bin/INTRODUCTION.md deleted file mode 100644 index 356d8f54..00000000 --- a/templates/extended/bin/INTRODUCTION.md +++ /dev/null @@ -1,7 +0,0 @@ -# Adding executable files - -If you need your robot to have standalone binaries or executables, you -should add them to this directory. - -This directory will be in PATH when the robot is executed in Robocorp App -or through Robocorp CLI. diff --git a/templates/extended/conda.yaml b/templates/extended/conda.yaml new file mode 100644 index 00000000..231b1bf2 --- /dev/null +++ b/templates/extended/conda.yaml @@ -0,0 +1,15 @@ +channels: + # Define conda channels here. + - conda-forge + +dependencies: + # Define conda packages here. + # If available, always prefer the conda version of a package, installation will be faster and more efficient. + # https://anaconda.org/search + - python=3.9.13 + + - pip=22.1.2 + - pip: + # Define pip packages here. + # https://pypi.org/ + - rpaframework==15.6.0 # https://rpaframework.org/releasenotes.html diff --git a/templates/extended/config/INTRODUCTION.md b/templates/extended/config/INTRODUCTION.md deleted file mode 100644 index 38a19815..00000000 --- a/templates/extended/config/INTRODUCTION.md +++ /dev/null @@ -1,14 +0,0 @@ -# Dependencies (Python packages) - -If your robot project requires dependencies such as Python packages, -you should define them in the `conda.yaml` file in this directory. Robocorp -App will use `conda.yaml` file to set up a conda environment when executed -in a target environment. - -For a local environment, you can use `pip` or `conda` or another preferred -package manager to install the dependencies. Keep the `conda.yaml` in sync with -the required dependencies. Otherwise, the robot might work when run locally, -but not when run using Robocorp App. - -If you do not want to use conda at all and want to provide the execution -environment yourself, you can delete the `conda.yaml` file. diff --git a/templates/extended/config/conda.yaml b/templates/extended/config/conda.yaml deleted file mode 100644 index b8f48500..00000000 --- a/templates/extended/config/conda.yaml +++ /dev/null @@ -1,8 +0,0 @@ -channels: - - defaults - - conda-forge -dependencies: - - python=3.7.5 - - pip=20.1 - - pip: - - rpaframework==6.* diff --git a/templates/extended/devdata/INTRODUCTION.md b/templates/extended/devdata/INTRODUCTION.md deleted file mode 100644 index bc550df1..00000000 --- a/templates/extended/devdata/INTRODUCTION.md +++ /dev/null @@ -1,12 +0,0 @@ -# Devdata placeholder - -This directory should contain local development data. - -This data should not be used in actual Robocorp App run but can be -used when you are building and debugging your robots and tasks. - -## What to put here? - -- Local work item data -- Local environment variables (in JSON format) -- Local custom data files (*.csv, etc.) diff --git a/templates/extended/devdata/env.json b/templates/extended/devdata/env.json index e3cd832d..231725b1 100644 --- a/templates/extended/devdata/env.json +++ b/templates/extended/devdata/env.json @@ -1,5 +1,3 @@ { - "GREETINGS": "Hello", - "WHO_TO_GREET": "World", - "THE_ANSWER": 42 + "SOME_DEV_ENV_URL": "https://robocorp.com/docs/development-howtos/variables-and-secrets/" } diff --git a/templates/extended/keywords/keywords.robot b/templates/extended/keywords/keywords.robot new file mode 100644 index 00000000..11941193 --- /dev/null +++ b/templates/extended/keywords/keywords.robot @@ -0,0 +1,7 @@ +*** Settings *** +Documentation Template keyword resource. + + +*** Keywords *** +Example keyword + Log Today is ${TODAY} diff --git a/templates/extended/libraries/ExampleLibrary.py b/templates/extended/libraries/ExampleLibrary.py deleted file mode 100644 index 7036f665..00000000 --- a/templates/extended/libraries/ExampleLibrary.py +++ /dev/null @@ -1,6 +0,0 @@ -from datetime import date - - -class ExampleLibrary: - def current_date(self): - return date.today() diff --git a/templates/extended/libraries/INTRODUCTION.md b/templates/extended/libraries/INTRODUCTION.md deleted file mode 100644 index 11fd3458..00000000 --- a/templates/extended/libraries/INTRODUCTION.md +++ /dev/null @@ -1,5 +0,0 @@ -# Libraries - -Place your libraries in this directory. - -This directory will be inserted into PYTHONPATH environment variable. diff --git a/templates/extended/libraries/MyLibrary.py b/templates/extended/libraries/MyLibrary.py new file mode 100644 index 00000000..5f993eea --- /dev/null +++ b/templates/extended/libraries/MyLibrary.py @@ -0,0 +1,8 @@ +from robot.api import logger + + +class MyLibrary: + """Give this library a proper name and document it.""" + + def example_python_keyword(self): + logger.info("This is Python!") diff --git a/templates/extended/resources/INTRODUCTION.md b/templates/extended/resources/INTRODUCTION.md deleted file mode 100644 index 2c53f682..00000000 --- a/templates/extended/resources/INTRODUCTION.md +++ /dev/null @@ -1,5 +0,0 @@ -# Robot Resources - -Place your Robot Framework keyword files (`*.robot`) in this directory. - -This directory will be inserted into PYTHONPATH environment variable. diff --git a/templates/extended/resources/keywords.robot b/templates/extended/resources/keywords.robot deleted file mode 100644 index 689283e4..00000000 --- a/templates/extended/resources/keywords.robot +++ /dev/null @@ -1,12 +0,0 @@ -*** Settings *** -Library ExampleLibrary -Variables variables.py - -*** Keywords *** -Process greeting for today - ${current_date}= Current date - ${day_name}= Get week day name - Log Hello! Today is ${day_name}. The date is ${current_date}. console=True - -Get week day name - [Return] ${WEEK_DAY_NAME} diff --git a/templates/extended/robot.yaml b/templates/extended/robot.yaml old mode 100644 new mode 100755 index f40cabe1..0cd1d454 --- a/templates/extended/robot.yaml +++ b/templates/extended/robot.yaml @@ -1,25 +1,17 @@ tasks: - Default: - command: - - python - - -m - - robot - - --report - - NONE - - --outputdir - - output - - --logtitle - - Task log - - tasks/ + Run all tasks: + shell: python -m robot --report NONE --outputdir output --logtitle "Task log" tasks.robot + + Run Example task: + robotTaskName: Example Task -condaConfigFile: config/conda.yaml +condaConfigFile: conda.yaml ignoreFiles: - .gitignore artifactsDir: output PATH: - - bin - - entrypoints + - . PYTHONPATH: - - variables + - keywords - libraries - - resources + - variables diff --git a/templates/extended/tasks.robot b/templates/extended/tasks.robot new file mode 100644 index 00000000..91a4374d --- /dev/null +++ b/templates/extended/tasks.robot @@ -0,0 +1,15 @@ +*** Settings *** +Documentation Template robot main suite. +Library Collections +Library MyLibrary +Resource keywords.robot +Variables MyVariables.py + + +*** Tasks *** +Example Task + Example Keyword + Example Python Keyword + Log ${TODAY} + + diff --git a/templates/extended/tasks/INTRODUCTION.md b/templates/extended/tasks/INTRODUCTION.md deleted file mode 100644 index 54ec77ba..00000000 --- a/templates/extended/tasks/INTRODUCTION.md +++ /dev/null @@ -1,5 +0,0 @@ -# Robot tasks - -Place your Robot Framework task files (`*.robot`) in this directory. - -This directory will be inserted into PYTHONPATH environment variable. diff --git a/templates/extended/tasks/tasks.robot b/templates/extended/tasks/tasks.robot deleted file mode 100644 index 3c2b5a9f..00000000 --- a/templates/extended/tasks/tasks.robot +++ /dev/null @@ -1,7 +0,0 @@ -*** Settings *** -Documentation An example robot. -Resource keywords.robot - -*** Tasks *** -Log greeting for today - Process greeting for today diff --git a/templates/extended/variables/INTRODUCTION.md b/templates/extended/variables/INTRODUCTION.md deleted file mode 100644 index 509839f4..00000000 --- a/templates/extended/variables/INTRODUCTION.md +++ /dev/null @@ -1,5 +0,0 @@ -# Variables - -Place your variable files (`*.py`) in this directory. - -This directory will be inserted into PYTHONPATH environment variable. diff --git a/templates/extended/variables/MyVariables.py b/templates/extended/variables/MyVariables.py new file mode 100644 index 00000000..d398aff1 --- /dev/null +++ b/templates/extended/variables/MyVariables.py @@ -0,0 +1,3 @@ +from datetime import datetime + +TODAY = datetime.now() diff --git a/templates/extended/variables/__init__.py b/templates/extended/variables/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/templates/extended/variables/variables.py b/templates/extended/variables/variables.py index ca0fb586..d398aff1 100644 --- a/templates/extended/variables/variables.py +++ b/templates/extended/variables/variables.py @@ -1,7 +1,3 @@ -''' -Variables for Robot Framework goes here. -''' -import calendar -from datetime import date +from datetime import datetime -WEEK_DAY_NAME = calendar.day_name[date.today().weekday()] +TODAY = datetime.now() diff --git a/templates/python/conda.yaml b/templates/python/conda.yaml index b8f48500..231b1bf2 100644 --- a/templates/python/conda.yaml +++ b/templates/python/conda.yaml @@ -1,8 +1,15 @@ channels: - - defaults + # Define conda channels here. - conda-forge + dependencies: - - python=3.7.5 - - pip=20.1 + # Define conda packages here. + # If available, always prefer the conda version of a package, installation will be faster and more efficient. + # https://anaconda.org/search + - python=3.9.13 + + - pip=22.1.2 - pip: - - rpaframework==6.* + # Define pip packages here. + # https://pypi.org/ + - rpaframework==15.6.0 # https://rpaframework.org/releasenotes.html diff --git a/templates/python/robot.yaml b/templates/python/robot.yaml index 5d3610ca..f78b7601 100644 --- a/templates/python/robot.yaml +++ b/templates/python/robot.yaml @@ -1,8 +1,6 @@ tasks: - Default: - command: - - python - - task.py + Run Python: + shell: python task.py condaConfigFile: conda.yaml artifactsDir: output @@ -11,4 +9,4 @@ PATH: PYTHONPATH: - . ignoreFiles: - - .gitignore + - .gitignore diff --git a/templates/python/task.py b/templates/python/task.py index c08134a8..0919c9d0 100644 --- a/templates/python/task.py +++ b/templates/python/task.py @@ -1,4 +1,4 @@ -""" An example robot. """ +"""Template robot with Python.""" def minimal_task(): diff --git a/templates/standard/conda.yaml b/templates/standard/conda.yaml index b8f48500..231b1bf2 100644 --- a/templates/standard/conda.yaml +++ b/templates/standard/conda.yaml @@ -1,8 +1,15 @@ channels: - - defaults + # Define conda channels here. - conda-forge + dependencies: - - python=3.7.5 - - pip=20.1 + # Define conda packages here. + # If available, always prefer the conda version of a package, installation will be faster and more efficient. + # https://anaconda.org/search + - python=3.9.13 + + - pip=22.1.2 - pip: - - rpaframework==6.* + # Define pip packages here. + # https://pypi.org/ + - rpaframework==15.6.0 # https://rpaframework.org/releasenotes.html diff --git a/templates/standard/robot.yaml b/templates/standard/robot.yaml index f1f8613a..5c3786e2 100644 --- a/templates/standard/robot.yaml +++ b/templates/standard/robot.yaml @@ -1,16 +1,6 @@ tasks: - Default: - command: - - python - - -m - - robot - - --report - - NONE - - --outputdir - - output - - --logtitle - - Task log - - tasks.robot + Run all tasks: + shell: python -m robot --report NONE --outputdir output --logtitle "Task log" tasks.robot condaConfigFile: conda.yaml artifactsDir: output @@ -19,4 +9,4 @@ PATH: PYTHONPATH: - . ignoreFiles: - - .gitignore + - .gitignore diff --git a/templates/standard/tasks.robot b/templates/standard/tasks.robot index 6b74ec9e..4903b4cf 100644 --- a/templates/standard/tasks.robot +++ b/templates/standard/tasks.robot @@ -1,5 +1,6 @@ *** Settings *** -Documentation An example robot. +Documentation Template robot main suite. + *** Tasks *** Minimal task diff --git a/wizard/altcreate.go b/wizard/altcreate.go deleted file mode 100644 index 627ad954..00000000 --- a/wizard/altcreate.go +++ /dev/null @@ -1,117 +0,0 @@ -package wizard - -import ( - "bufio" - "errors" - "fmt" - "os" - "path/filepath" - "regexp" - "strconv" - "strings" - - "github.com/robocorp/rcc/common" - "github.com/robocorp/rcc/operations" - "github.com/robocorp/rcc/pathlib" - "github.com/robocorp/rcc/pretty" -) - -const ( - newline = '\n' -) - -var ( - namePattern = regexp.MustCompile("^[\\w-]*$") - digitPattern = regexp.MustCompile("^\\d+$") -) - -func ask(question, defaults string, validator *regexp.Regexp, erratic string) (string, error) { - for { - common.Stdout("%s? %s%s %s[%s]:%s ", pretty.Green, pretty.White, question, pretty.Grey, defaults, pretty.Reset) - source := bufio.NewReader(os.Stdin) - reply, err := source.ReadString(newline) - common.Stdout("\n") - if err != nil { - return "", err - } - reply = strings.TrimSpace(reply) - if !validator.MatchString(reply) { - common.Stdout("%s%s%s\n\n", pretty.Red, erratic, pretty.Reset) - continue - } - if len(reply) == 0 { - return defaults, nil - } - return reply, nil - } -} - -func choose(question, label string, candidates []string) (string, error) { - keys := []string{} - common.Stdout("%s%s:%s\n", pretty.Grey, label, pretty.Reset) - for index, candidate := range candidates { - key := index + 1 - keys = append(keys, fmt.Sprintf("%d", key)) - common.Stdout(" %s%2d: %s%s%s\n", pretty.Grey, key, pretty.White, candidate, pretty.Reset) - } - common.Stdout("\n") - selectable := strings.Join(keys, "|") - pattern, err := regexp.Compile(fmt.Sprintf("^(?:%s)?$", selectable)) - if err != nil { - return "", err - } - reply, err := ask(question, "1", pattern, "Give selections number from above list.") - if err != nil { - return "", err - } - selected, err := strconv.Atoi(reply) - if err != nil { - return "", err - } - return candidates[selected-1], nil -} - -func AltCreate(arguments []string) error { - common.Stdout("\n") - - warning(len(arguments) > 1, "You provided more than one argument, but only the first one will be\nused as the name.") - robotName, err := ask("Give robot name", firstOf(arguments, "my-first-robot"), namePattern, "Use just normal english characters and no spaces!") - - if err != nil { - return err - } - - fullpath, err := filepath.Abs(robotName) - if err != nil { - return err - } - - if pathlib.IsDir(fullpath) { - return errors.New(fmt.Sprintf("Folder %s already exists. Try with other name.", robotName)) - } - - selected, err := choose("Choose a template", "Templates", operations.ListTemplates()) - if err != nil { - return err - } - - common.Stdout("%sCreating the %s%s%s robot: %s%s%s\n", pretty.White, pretty.Cyan, selected, pretty.White, pretty.Cyan, robotName, pretty.Reset) - common.Stdout("\n") - - err = operations.InitializeWorkarea(fullpath, selected, false) - if err != nil { - return err - } - - common.Stdout("%s%s%sThe %s robot has been created to: %s%s\n", pretty.Yellow, pretty.Sparkles, pretty.Green, selected, robotName, pretty.Reset) - common.Stdout("\n") - - common.Stdout("%s%sGet started with following commands:%s\n", pretty.White, pretty.Rocket, pretty.Reset) - common.Stdout("\n") - - common.Stdout("%s$ %scd %s%s\n", pretty.Grey, pretty.Cyan, robotName, pretty.Reset) - common.Stdout("%s$ %srcc run%s\n", pretty.Grey, pretty.Cyan, pretty.Reset) - common.Stdout("\n") - - return nil -} diff --git a/wizard/common.go b/wizard/common.go index 9be8505d..ec2237be 100644 --- a/wizard/common.go +++ b/wizard/common.go @@ -1,15 +1,51 @@ package wizard import ( - "errors" + "bufio" + "fmt" + "os" + "regexp" "strings" "github.com/robocorp/rcc/common" "github.com/robocorp/rcc/pretty" ) +const ( + UNIX_NEWLINE = "\n" + WINDOWS_NEWLINE = "\r\n" +) + +var ( + namePattern = regexp.MustCompile("^[\\w-]*$") +) + +type Validator func(string) bool + type WizardFn func([]string) error +func memberValidation(members []string, erratic string) Validator { + return func(input string) bool { + for _, member := range members { + if input == member { + return true + } + } + common.Stdout("%s%s%s\n\n", pretty.Red, erratic, pretty.Reset) + return false + } +} + +func regexpValidation(validator *regexp.Regexp, erratic string) Validator { + return func(input string) bool { + if !validator.MatchString(input) { + common.Stdout("%s%s%s\n\n", pretty.Red, erratic, pretty.Reset) + return false + } + return true + } +} + func warning(condition bool, message string) { if condition { common.Stdout("%s%s%s\n\n", pretty.Yellow, message, pretty.Reset) @@ -23,16 +59,27 @@ func firstOf(arguments []string, missing string) string { return missing } -func ifThenElse(condition bool, truthy, falsy string) string { - if condition { - return truthy - } - return falsy +func note(form string, details ...interface{}) { + message := fmt.Sprintf(form, details...) + common.Stdout("%s! %s%s%s\n", pretty.Red, pretty.White, message, pretty.Reset) } -func hasLength(value string) error { - if len(strings.TrimSpace(value)) > 2 { - return nil +func ask(question, defaults string, validator Validator) (string, error) { + for { + common.Stdout("%s? %s%s %s[%s]:%s ", pretty.Green, pretty.White, question, pretty.Grey, defaults, pretty.Reset) + source := bufio.NewReader(os.Stdin) + reply, err := source.ReadString(newline) + common.Stdout("\n") + if err != nil { + return "", err + } + if reply == UNIX_NEWLINE || reply == WINDOWS_NEWLINE { + reply = defaults + } + reply = strings.TrimSpace(reply) + if !validator(reply) { + continue + } + return reply, nil } - return errors.New("Value too short!") } diff --git a/wizard/create.go b/wizard/create.go index e37165eb..26079b1a 100644 --- a/wizard/create.go +++ b/wizard/create.go @@ -1,28 +1,57 @@ package wizard import ( - "errors" "fmt" "path/filepath" + "regexp" + "sort" + "strconv" + "strings" - "github.com/manifoldco/promptui" "github.com/robocorp/rcc/common" "github.com/robocorp/rcc/operations" "github.com/robocorp/rcc/pathlib" "github.com/robocorp/rcc/pretty" ) +const ( + newline = '\n' +) + +var ( + digitPattern = regexp.MustCompile("^\\d+$") +) + +func choose(question, label string, candidates []string) (string, error) { + keys := []string{} + common.Stdout("%s%s:%s\n", pretty.Grey, label, pretty.Reset) + for index, candidate := range candidates { + key := index + 1 + keys = append(keys, fmt.Sprintf("%d", key)) + common.Stdout(" %s%2d: %s%s%s\n", pretty.Grey, key, pretty.White, candidate, pretty.Reset) + } + common.Stdout("\n") + selectable := strings.Join(keys, "|") + pattern, err := regexp.Compile(fmt.Sprintf("^(?:%s)?$", selectable)) + if err != nil { + return "", err + } + reply, err := ask(question, "1", regexpValidation(pattern, "Give selections number from above list.")) + if err != nil { + return "", err + } + selected, err := strconv.Atoi(reply) + if err != nil { + return "", err + } + return candidates[selected-1], nil +} + func Create(arguments []string) error { common.Stdout("\n") warning(len(arguments) > 1, "You provided more than one argument, but only the first one will be\nused as the name.") - prompt := promptui.Prompt{ - Label: "Give robot name", - Default: firstOf(arguments, "my-first-robot"), - Validate: hasLength, - } - robotName, err := prompt.Run() - common.Stdout("\n") + robotName, err := ask("Give robot name", firstOf(arguments, "my-first-robot"), regexpValidation(namePattern, "Use just normal english word characters and no spaces!")) if err != nil { return err @@ -34,30 +63,28 @@ func Create(arguments []string) error { } if pathlib.IsDir(fullpath) { - return errors.New(fmt.Sprintf("Folder %s already exists. Try with other name.", robotName)) + return fmt.Errorf("Folder %s already exists. Try with other name.", robotName) } - selection := promptui.Select{ - Label: "Choose a template", - Items: operations.ListTemplates(), + templates := operations.ListTemplatesWithDescription(false) + descriptions := make([]string, 0, len(templates)) + lookup := make(map[string]string) + for _, template := range templates { + descriptions = append(descriptions, template[1]) + lookup[template[1]] = template[0] } - - _, selected, err := selection.Run() - common.Stdout("\n") - + sort.Strings(descriptions) + selected, err := choose("Choose a template", "Templates", descriptions) if err != nil { return err } - common.Stdout("%sCreating the %s%s%s robot: %s%s%s\n", pretty.White, pretty.Cyan, selected, pretty.White, pretty.Cyan, robotName, pretty.Reset) - common.Stdout("\n") - - err = operations.InitializeWorkarea(fullpath, selected, false) + err = operations.InitializeWorkarea(fullpath, lookup[selected], false, false) if err != nil { return err } - common.Stdout("%s%s%sThe %s robot has been created to: %s%s\n", pretty.Yellow, pretty.Sparkles, pretty.Green, selected, robotName, pretty.Reset) + common.Stdout("%s%s%sThe %s%s%s robot has been created to: %s%s%s\n", pretty.Yellow, pretty.Sparkles, pretty.Green, pretty.Cyan, selected, pretty.Green, pretty.Cyan, robotName, pretty.Reset) common.Stdout("\n") common.Stdout("%s%sGet started with following commands:%s\n", pretty.White, pretty.Rocket, pretty.Reset) @@ -65,6 +92,7 @@ func Create(arguments []string) error { common.Stdout("%s$ %scd %s%s\n", pretty.Grey, pretty.Cyan, robotName, pretty.Reset) common.Stdout("%s$ %srcc run%s\n", pretty.Grey, pretty.Cyan, pretty.Reset) + common.Stdout("%s# or with name in case of multiple tasks in one robot\n$ %srcc run --task \"\"%s\n", pretty.Grey, pretty.Cyan, pretty.Reset) common.Stdout("\n") return nil diff --git a/wizard/wizard_test.go b/wizard/wizard_test.go new file mode 100644 index 00000000..5fcd4e55 --- /dev/null +++ b/wizard/wizard_test.go @@ -0,0 +1 @@ +package wizard_test diff --git a/xviper/runminutes.go b/xviper/runminutes.go deleted file mode 100644 index cd21dc91..00000000 --- a/xviper/runminutes.go +++ /dev/null @@ -1,25 +0,0 @@ -package xviper - -import ( - "math" - "time" -) - -const ( - runMinutesStats = `stats.rccminutes` -) - -type runMarker time.Time - -func RunMinutes() runMarker { - return runMarker(time.Now()) -} - -func (it runMarker) Done() uint64 { - delta := time.Now().Sub(time.Time(it)) - minutes := uint64(math.Max(1.0, math.Ceil(delta.Minutes()))) - previous := GetUint64(runMinutesStats) - total := previous + minutes - Set(runMinutesStats, total) - return total -} diff --git a/xviper/runminutes_test.go b/xviper/runminutes_test.go deleted file mode 100644 index 5554d4a2..00000000 --- a/xviper/runminutes_test.go +++ /dev/null @@ -1,22 +0,0 @@ -package xviper_test - -import ( - "testing" - "time" - - "github.com/robocorp/rcc/hamlet" - "github.com/robocorp/rcc/xviper" -) - -func TestCanCreateRunMinutes(t *testing.T) { - must_be, wont_be := hamlet.Specifications(t) - - sut := xviper.RunMinutes() - wont_be.Nil(sut) - time.Sleep(100 * time.Millisecond) - first := sut.Done() - must_be.True(first > 0) - second := xviper.RunMinutes().Done() - must_be.True(second > first) - must_be.Equal(first+1, second) -} diff --git a/xviper/tracking.go b/xviper/tracking.go index a3f4f474..2eb6158f 100644 --- a/xviper/tracking.go +++ b/xviper/tracking.go @@ -6,6 +6,8 @@ import ( "math/rand" "strings" "time" + + "github.com/robocorp/rcc/common" ) const ( @@ -18,7 +20,7 @@ var ( ) func init() { - rand.Seed(time.Now().Unix()) + rand.Seed(common.When) } func AsGuid(content []byte) string { @@ -53,5 +55,5 @@ func ConsentTracking(state bool) { } func CanTrack() bool { - return GetBool(trackingConsentKey) + return GetBool(trackingConsentKey) && !common.WarrantyVoided() } diff --git a/xviper/wrapper.go b/xviper/wrapper.go index 30eed409..f536ba14 100644 --- a/xviper/wrapper.go +++ b/xviper/wrapper.go @@ -35,25 +35,37 @@ type config struct { } func (it *config) Save() { + if common.WarrantyVoided() { + return + } if len(it.Filename) == 0 { return } - locker, err := pathlib.Locker(it.Lockfile, 125) + completed := pathlib.LockWaitMessage(it.Lockfile, "Serialized config access [config lock]") + locker, err := pathlib.Locker(it.Lockfile, 125, false) + completed() if err != nil { common.Log("FATAL: could not lock %v, reason %v; ignored.", it.Lockfile, err) return } defer locker.Release() - it.Viper.WriteConfigAs(it.Filename) + err = it.Viper.WriteConfigAs(it.Filename) + if err != nil { + common.Log("FATAL: could not write %v, reason %v; ignored.", it.Filename, err) + return + } + defer pathlib.RestrictOwnerOnly(it.Filename) when, err := pathlib.Modtime(it.Filename) if err == nil { it.Timestamp = when } } -func (it *config) Reload() { - locker, err := pathlib.Locker(it.Lockfile, 125) +func (it *config) reload() { + completed := pathlib.LockWaitMessage(it.Lockfile, "Serialized config access [config lock]") + locker, err := pathlib.Locker(it.Lockfile, 125, false) + completed() if err != nil { common.Log("FATAL: could not lock %v, reason %v; ignored.", it.Lockfile, err) return @@ -62,6 +74,7 @@ func (it *config) Reload() { it.Viper = viper.New() it.Viper.SetConfigFile(it.Filename) + defer pathlib.RestrictOwnerOnly(it.Filename) err = it.Viper.ReadInConfig() var when time.Time if err == nil { @@ -77,7 +90,7 @@ func (it *config) Reload() { func (it *config) Reset(filename string) { it.Filename = filename it.Lockfile = fmt.Sprintf("%s.lck", filename) - it.Reload() + it.reload() } func (it *config) Summon() *viper.Viper { @@ -90,7 +103,7 @@ func (it *config) Summon() *viper.Viper { } if when.After(it.Timestamp) { common.Debug("Configuration %v changed, reloading!", it.Filename) - it.Reload() + it.reload() } return it.Viper } @@ -103,6 +116,10 @@ func runner(todo <-chan command) { } } +func IsAvailable() bool { + return len(AllKeys()) > 0 +} + func SetConfigFile(in string) { pipeline <- func(core *config) { core.Reset(in) @@ -120,6 +137,14 @@ func Set(key string, value interface{}) { <-flow } +func Lockfile() string { + flow := make(chan string) + pipeline <- func(core *config) { + flow <- core.Lockfile + } + return <-flow +} + func ConfigFileUsed() string { flow := make(chan string) pipeline <- func(core *config) { @@ -160,14 +185,6 @@ func GetInt64(key string) int64 { return <-flow } -func GetInt(key string) int { - flow := make(chan int) - pipeline <- func(core *config) { - flow <- core.Summon().GetInt(key) - } - return <-flow -} - func GetString(key string) string { flow := make(chan string) pipeline <- func(core *config) {