From 672ba0ebab20f56ccb7051eecb999fb10035c391 Mon Sep 17 00:00:00 2001 From: Valery Piashchynski Date: Thu, 27 Jun 2024 12:20:02 +0200 Subject: [PATCH] Initial Commit Signed-off-by: Valery Piashchynski --- .githooks/pre-commit | 11 + .github/CODEOWNERS | 3 + .github/FUNDING.yml | 3 + .github/ISSUE_TEMPLATE/bug-report.yml | 42 + .github/ISSUE_TEMPLATE/config.yml | 6 + .github/ISSUE_TEMPLATE/feature-request.yml | 38 + .github/dependabot.yml | 29 + .github/pull_request_template.md | 24 + .github/workflows/codeql-analysis.yml | 73 ++ .github/workflows/linters.yml | 23 + .github/workflows/semgrep.yml | 15 + .github/workflows/tests.yml | 61 + .gitignore | 38 +- .golangci.yml | 81 ++ LICENSE | 2 +- Makefile | 24 + README.md | 23 + fsm/fsm.go | 158 +++ fsm/state.go | 31 + fsm/state_test.go | 32 + githooks-installer.sh | 7 + go.mod | 27 + go.sum | 43 + internal/protocol.go | 94 ++ ipc/pipe/pipe.go | 146 +++ ipc/pipe/pipe_spawn_test.go | 414 +++++++ ipc/pipe/pipe_test.go | 467 ++++++++ ipc/socket/socket.go | 193 ++++ ipc/socket/socket_spawn_test.go | 521 +++++++++ ipc/socket/socket_test.go | 567 ++++++++++ payload/payload.go | 27 + pool/allocator.go | 82 ++ pool/command.go | 8 + pool/config.go | 83 ++ pool/static_pool/debug.go | 147 +++ pool/static_pool/fuzz_test.go | 45 + pool/static_pool/options.go | 19 + pool/static_pool/pool.go | 418 +++++++ pool/static_pool/pool_test.go | 1173 ++++++++++++++++++++ pool/static_pool/stream.go | 31 + pool/static_pool/supervisor.go | 180 +++ pool/static_pool/supervisor_test.go | 720 ++++++++++++ process/isolate.go | 59 + process/isolate_windows.go | 17 + state/process/state.go | 53 + tests/allocate-failed.php | 18 + tests/broken.php | 14 + tests/client.php | 35 + tests/composer.json | 21 + tests/crc_error.php | 17 + tests/delay.php | 18 + tests/echo.php | 17 + tests/error.php | 13 + tests/exec_ttl.php | 15 + tests/failboot.php | 3 + tests/gzip-large-file.txt | 19 + tests/head.php | 17 + tests/http/client.php | 51 + tests/http/cookie.php | 334 ++++++ tests/http/data.php | 17 + tests/http/echo.php | 10 + tests/http/echoDelay.php | 11 + tests/http/echoerr.php | 12 + tests/http/env.php | 10 + tests/http/error.php | 9 + tests/http/error2.php | 9 + tests/http/header.php | 11 + tests/http/headers.php | 11 + tests/http/ip.php | 11 + tests/http/memleak.php | 11 + tests/http/payload.php | 18 + tests/http/pid.php | 11 + tests/http/push.php | 10 + tests/http/request-uri.php | 10 + tests/http/server.php | 11 + tests/http/slow-client.php | 52 + tests/http/stuck.php | 11 + tests/http/upload.php | 35 + tests/http/user-agent.php | 10 + tests/idle.php | 15 + tests/issue659.php | 21 + tests/memleak.php | 15 + tests/metrics-issue-571.php | 37 + tests/pid.php | 17 + tests/pipes_test_script.sh | 2 + tests/psr-worker-bench.php | 30 + tests/psr-worker-post.php | 30 + tests/psr-worker-slow.php | 29 + tests/psr-worker.php | 28 + tests/raw-error.php | 21 + tests/sample.txt | 1 + tests/should-not-be-killed.php | 19 + tests/sleep-ttl.php | 15 + tests/sleep.php | 15 + tests/sleep_short.php | 15 + tests/slow-client.php | 38 + tests/slow-destroy.php | 37 + tests/slow-pid.php | 18 + tests/socket_test_script.sh | 2 + tests/src/Activity/SimpleActivity.php | 63 ++ tests/src/Client/StartNewWorkflow.php | 23 + tests/src/Workflow/SagaWorkflow.php | 54 + tests/stop.php | 25 + tests/stream_worker.php | 34 + tests/supervised.php | 14 + worker/options.go | 49 + worker/worker.go | 674 +++++++++++ worker/worker_test.go | 45 + worker_watcher/container/channel/vec.go | 139 +++ worker_watcher/worker_watcher.go | 431 +++++++ 110 files changed, 9087 insertions(+), 9 deletions(-) create mode 100644 .githooks/pre-commit create mode 100644 .github/CODEOWNERS create mode 100644 .github/FUNDING.yml create mode 100644 .github/ISSUE_TEMPLATE/bug-report.yml create mode 100644 .github/ISSUE_TEMPLATE/config.yml create mode 100644 .github/ISSUE_TEMPLATE/feature-request.yml create mode 100644 .github/dependabot.yml create mode 100644 .github/pull_request_template.md create mode 100644 .github/workflows/codeql-analysis.yml create mode 100644 .github/workflows/linters.yml create mode 100644 .github/workflows/semgrep.yml create mode 100644 .github/workflows/tests.yml create mode 100755 .golangci.yml create mode 100644 Makefile create mode 100644 README.md create mode 100644 fsm/fsm.go create mode 100755 fsm/state.go create mode 100755 fsm/state_test.go create mode 100644 githooks-installer.sh create mode 100644 go.mod create mode 100644 go.sum create mode 100755 internal/protocol.go create mode 100755 ipc/pipe/pipe.go create mode 100644 ipc/pipe/pipe_spawn_test.go create mode 100644 ipc/pipe/pipe_test.go create mode 100755 ipc/socket/socket.go create mode 100644 ipc/socket/socket_spawn_test.go create mode 100755 ipc/socket/socket_test.go create mode 100644 payload/payload.go create mode 100644 pool/allocator.go create mode 100644 pool/command.go create mode 100644 pool/config.go create mode 100644 pool/static_pool/debug.go create mode 100644 pool/static_pool/fuzz_test.go create mode 100644 pool/static_pool/options.go create mode 100644 pool/static_pool/pool.go create mode 100644 pool/static_pool/pool_test.go create mode 100644 pool/static_pool/stream.go create mode 100644 pool/static_pool/supervisor.go create mode 100644 pool/static_pool/supervisor_test.go create mode 100755 process/isolate.go create mode 100755 process/isolate_windows.go create mode 100644 state/process/state.go create mode 100644 tests/allocate-failed.php create mode 100644 tests/broken.php create mode 100644 tests/client.php create mode 100644 tests/composer.json create mode 100644 tests/crc_error.php create mode 100644 tests/delay.php create mode 100644 tests/echo.php create mode 100644 tests/error.php create mode 100644 tests/exec_ttl.php create mode 100644 tests/failboot.php create mode 100644 tests/gzip-large-file.txt create mode 100644 tests/head.php create mode 100644 tests/http/client.php create mode 100644 tests/http/cookie.php create mode 100644 tests/http/data.php create mode 100644 tests/http/echo.php create mode 100644 tests/http/echoDelay.php create mode 100644 tests/http/echoerr.php create mode 100644 tests/http/env.php create mode 100644 tests/http/error.php create mode 100644 tests/http/error2.php create mode 100644 tests/http/header.php create mode 100644 tests/http/headers.php create mode 100644 tests/http/ip.php create mode 100644 tests/http/memleak.php create mode 100644 tests/http/payload.php create mode 100644 tests/http/pid.php create mode 100644 tests/http/push.php create mode 100644 tests/http/request-uri.php create mode 100644 tests/http/server.php create mode 100644 tests/http/slow-client.php create mode 100644 tests/http/stuck.php create mode 100644 tests/http/upload.php create mode 100644 tests/http/user-agent.php create mode 100644 tests/idle.php create mode 100644 tests/issue659.php create mode 100644 tests/memleak.php create mode 100644 tests/metrics-issue-571.php create mode 100644 tests/pid.php create mode 100755 tests/pipes_test_script.sh create mode 100644 tests/psr-worker-bench.php create mode 100644 tests/psr-worker-post.php create mode 100644 tests/psr-worker-slow.php create mode 100644 tests/psr-worker.php create mode 100644 tests/raw-error.php create mode 100644 tests/sample.txt create mode 100644 tests/should-not-be-killed.php create mode 100644 tests/sleep-ttl.php create mode 100644 tests/sleep.php create mode 100644 tests/sleep_short.php create mode 100644 tests/slow-client.php create mode 100644 tests/slow-destroy.php create mode 100644 tests/slow-pid.php create mode 100755 tests/socket_test_script.sh create mode 100644 tests/src/Activity/SimpleActivity.php create mode 100644 tests/src/Client/StartNewWorkflow.php create mode 100644 tests/src/Workflow/SagaWorkflow.php create mode 100644 tests/stop.php create mode 100644 tests/stream_worker.php create mode 100644 tests/supervised.php create mode 100644 worker/options.go create mode 100644 worker/worker.go create mode 100644 worker/worker_test.go create mode 100644 worker_watcher/container/channel/vec.go create mode 100644 worker_watcher/worker_watcher.go diff --git a/.githooks/pre-commit b/.githooks/pre-commit new file mode 100644 index 0000000..2d90586 --- /dev/null +++ b/.githooks/pre-commit @@ -0,0 +1,11 @@ +#!/bin/bash + +set -e -o pipefail + +# https://github.com/koalaman/shellcheck/wiki/SC2039#redirect-both-stdout-and-stderr +if ! command -v golangci-lint 2>&1 /dev/null; then + echo "golangci-lint is not installed" + exit 1 +fi + +exec golangci-lint --build-tags=race run "$@" diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..e43ff93 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,3 @@ +# Primary owners + +@wolfy-j @rustatian diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 0000000..38798a2 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,3 @@ +# These are supported funding model platforms + +github: roadrunner-server diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml new file mode 100644 index 0000000..35b5573 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug-report.yml @@ -0,0 +1,42 @@ +name: Bug Report +description: 🐛 File a bug report +title: "[🐛 BUG]: " +labels: ["B-bug", "F-need-verification"] +assignees: + - rustatian +body: + - type: markdown + attributes: + value: | + Thanks for taking the time to fill out this bug report! + + - type: checkboxes + id: search-done + attributes: + label: No duplicates đŸĨ˛. + options: + - label: I have searched for a similar issue in our bug tracker and didn't find any solutions. + required: true + - type: textarea + id: what-happened + attributes: + label: What happened? + description: Also tell us, what did you expect to happen? + placeholder: Tell us what you see! + value: "A bug happened!" + validations: + required: true + - type: textarea + id: version + attributes: + label: Version + description: What version of our software are you running? + placeholder: 2.6.0 + validations: + required: true + - type: textarea + id: logs + attributes: + label: Relevant log output + description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks. + render: shell \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..073e3f3 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,6 @@ +blank_issues_enabled: false + +contact_links: + - name: ❓ Start a discussion or ask a question. + url: https://github.com/roadrunner-server/roadrunner/discussions + about: Please ask and answer questions here. diff --git a/.github/ISSUE_TEMPLATE/feature-request.yml b/.github/ISSUE_TEMPLATE/feature-request.yml new file mode 100644 index 0000000..e26fde1 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature-request.yml @@ -0,0 +1,38 @@ +name: Feauture request +description: 💡 Suggest an idea for this project +title: "[💡 FEATURE REQUEST]: " +labels: ["C-feature-request"] +assignees: + - rustatian +body: + - type: markdown + attributes: + value: | + Thanks for taking the time to share your idea! + + - type: dropdown + id: plugin + attributes: + label: Plugin + description: What plugin is affected? + options: + - GRPC + - HTTP + - JOBS + - TCP + - File server + - Config + - KV + - Service + - Server + - Status + + - type: textarea + id: idea + attributes: + label: I have an idea! + description: Clear and concise description of your idea. + placeholder: Tell us what you see! + value: "I have an idea, listen to me!!" + validations: + required: true \ No newline at end of file diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..21db080 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,29 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates + +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates +version: 2 + +updates: + - package-ecosystem: gomod # See documentation for possible values + directory: "/" # Location of package manifests + schedule: + interval: daily + reviewers: + - "rustatian" + assignees: + - "rustatian" + + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: daily + reviewers: + - "rustatian" + assignees: + - "rustatian" diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..c346785 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,24 @@ +# Reason for This PR + +`[Author TODO: add issue # or explain reasoning.]` + +## Description of Changes + +`[Author TODO: add description of changes.]` + +## License Acceptance + +By submitting this pull request, I confirm that my contribution is made under +the terms of the MIT license. + +## PR Checklist + +`[Author TODO: Meet these criteria.]` +`[Reviewer TODO: Verify that these criteria are met. Request changes if not]` + +- [ ] All commits in this PR are signed (`git commit -s`). +- [ ] The reason for this PR is clearly provided (issue no. or explanation). +- [ ] The description of changes is clear and encompassing. +- [ ] Any required documentation changes (code and docs) are included in this PR. +- [ ] Any user-facing changes are mentioned in `CHANGELOG.md`. +- [ ] All added/changed functionality is tested. diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 0000000..0a5599e --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,73 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +name: "CodeQL" + +on: + push: + branches: [ master ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ master ] + schedule: + - cron: '0 15 * * 6' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + # Override automatic language detection by changing the below list + # Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', 'python'] + language: [ 'go' ] + # Learn more... + # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + # We must fetch at least the immediate parents so that if this is + # a pull request then we can checkout the head. + fetch-depth: 2 + + # Initializes the Golang environment for the CodeQL tools. + # https://github.com/github/codeql-action/issues/1842#issuecomment-1704398087 + - name: Install Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + # queries: ./path/to/local/query, your-org/your-repo/queries@main + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v3 + + # ℹī¸ Command-line programs to run using the OS shell. + # 📚 https://git.io/JvXDl + + # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following three lines + # and modify them (or add more) to build your code if your project + # uses a compiled language + + #- run: | + # make bootstrap + # make release + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 diff --git a/.github/workflows/linters.yml b/.github/workflows/linters.yml new file mode 100644 index 0000000..72d5d4d --- /dev/null +++ b/.github/workflows/linters.yml @@ -0,0 +1,23 @@ +name: Linters + +on: [push, pull_request] + +jobs: + golangci-lint: + name: Golang-CI (lint) + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 # action page: + with: + go-version: stable + + - name: Run linter + uses: golangci/golangci-lint-action@v6.0.1 # Action page: + with: + version: v1.59 # without patch version + only-new-issues: false # show only new issues if it's a pull request + args: --timeout=10m --build-tags=race diff --git a/.github/workflows/semgrep.yml b/.github/workflows/semgrep.yml new file mode 100644 index 0000000..dcade81 --- /dev/null +++ b/.github/workflows/semgrep.yml @@ -0,0 +1,15 @@ +name: semgrep + +on: [push, pull_request] + +jobs: + semgrep: + name: semgrep/ci + runs-on: ubuntu-latest + env: + SEMGREP_APP_TOKEN: ${{ secrets.SEMGREP_APP_TOKEN }} + container: + image: returntocorp/semgrep + steps: + - uses: actions/checkout@v4 + - run: semgrep ci diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..e34f8ce --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,61 @@ +name: tests + +on: [push, pull_request] + +jobs: + golang: + name: Build (Go ${{ matrix.go }}, PHP ${{ matrix.php }}, OS ${{matrix.os}}) + runs-on: ${{ matrix.os }} + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + php: ["8.3"] + go: [stable] + os: ["ubuntu-latest", "macos-latest", "windows-latest"] + steps: + - name: Set up Go ${{ matrix.go }} + uses: actions/setup-go@v5 # action page: + with: + go-version: ${{ matrix.go }} + + - name: Set up PHP ${{ matrix.php }} + uses: shivammathur/setup-php@v2 # action page: + with: + php-version: ${{ matrix.php }} + extensions: sockets + + - name: Check out code + uses: actions/checkout@v4 + + - name: Get Composer Cache Directory + id: composer-cache + run: echo "::set-output name=dir::$(composer config cache-files-dir)" + + - name: Init Composer Cache # Docs: + uses: actions/cache@v4 + with: + path: ${{ steps.composer-cache.outputs.dir }} + key: ${{ runner.os }}-composer-${{ matrix.php }}-${{ hashFiles('**/composer.json') }} + restore-keys: ${{ runner.os }}-composer- + + - name: Install Composer dependencies + run: cd tests && composer update --prefer-dist --no-progress --ansi + + - name: Init Go modules Cache # Docs: + uses: actions/cache@v4 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: ${{ runner.os }}-go- + + - name: Install Go dependencies + run: go mod download + + - name: Run golang tests with coverage + run: make test_coverage + + - uses: codecov/codecov-action@v4 # Docs: + with: + file: ./coverage-ci/summary.txt + fail_ci_if_error: false diff --git a/.gitignore b/.gitignore index 088ba6b..3ffd4cc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,32 @@ -# Generated by Cargo -# will have compiled files and executables -/target/ +# Created by .ignore support plugin (hsz.mobi) +### Go template +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib -# Remove Cargo.lock from gitignore if creating an executable, leave it for libraries -# More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html -Cargo.lock +# Test binary, built with `go test -c` +*.test +unit_tests +unit_tests_copied +dir1 +coverage-ci -# These are backup files generated by rustfmt -**/*.rs.bk +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Dependency directories (remove the comment below to include it) +# vendor/ +.idea +composer.lock +**/composer.lock +tests/composer.lock +vendor +builds/ +tests/vendor/ +.rr-sample.yaml +cmd +rr +**/old diff --git a/.golangci.yml b/.golangci.yml new file mode 100755 index 0000000..5edbd77 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,81 @@ +# Documentation: + +run: + timeout: 1m + modules-download-mode: readonly + allow-parallel-runners: true + +output: + formats: + - format: colored-line-number + +linters-settings: + revive: + confidence: 0.8 + godot: + scope: declarations + capital: true + dupl: + threshold: 100 + goconst: + min-len: 2 + min-occurrences: 3 + misspell: + locale: US + lll: + line-length: 120 + prealloc: + simple: true + range-loops: true + for-loops: true + nolintlint: + require-specific: true + +linters: # All available linters list: + disable-all: true + enable: + - asciicheck # Simple linter to check that your code does not contain non-ASCII identifiers + - bodyclose # Checks whether HTTP response body is closed successfully + - dogsled # Checks assignments with too many blank identifiers (e.g. x, _, _, _, := f()) + - errcheck # Errcheck is a program for checking for unchecked errors in go programs. These unchecked errors can be critical bugs in some cases + - exhaustive # check exhaustiveness of enum switch statements + - exportloopref # checks for pointers to enclosing loop variables + - gochecknoinits # Checks that no init functions are present in Go code + - goconst # Finds repeated strings that could be replaced by a constant + - gocritic # The most opinionated Go source code linter + - gofmt # Gofmt checks whether code was gofmt-ed. By default this tool runs with -s option to check for code simplification + - goimports # Goimports does everything that gofmt does. Additionally it checks unused imports + - revive # Golint differs from gofmt. Gofmt reformats Go source code, whereas golint prints out style mistakes + - goprintffuncname # Checks that printf-like functions are named with `f` at the end + - gosec # Inspects source code for security problems + - gosimple # Linter for Go source code that specializes in simplifying a code + - govet # Vet examines Go source code and reports suspicious constructs, such as Printf calls whose arguments do not align with the format string + - ineffassign # Detects when assignments to existing variables are not used + - misspell # Finds commonly misspelled English words in comments + - nakedret # Finds naked returns in functions greater than a specified function length + - noctx # finds sending http request without context.Context + - nolintlint # Reports ill-formed or insufficient nolint directives + - prealloc # Finds slice declarations that could potentially be preallocated + - staticcheck # Staticcheck is a go vet on steroids, applying a ton of static analysis checks + - stylecheck # Stylecheck is a replacement for golint + - tparallel # detects inappropriate usage of t.Parallel() method in your Go test codes + - unconvert # Remove unnecessary type conversions + - unused # Checks Go code for unused constants, variables, functions and types + - whitespace # Tool for detection of leading and trailing whitespace + +issues: + exclude-dirs: + - .github + - .git + exclude-rules: + - path: _test\.go + linters: + - dupl + - funlen + - scopelint + - gocognit + - goconst + - noctx + - gosimple + - revive + - gochecknoinits diff --git a/LICENSE b/LICENSE index 23cbd5d..be9e4bf 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2022 RoadRunner +Copyright (c) 2024 Spiral Scout Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..b15d3d1 --- /dev/null +++ b/Makefile @@ -0,0 +1,24 @@ +#!/usr/bin/make +# Makefile readme (ru): +# Makefile readme (en): + +SHELL = /bin/sh + +test_coverage: + rm -rf coverage-ci + mkdir ./coverage-ci + go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/pipe.out -covermode=atomic ./ipc/pipe + go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/socket.out -covermode=atomic ./ipc/socket + go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/pool_static.out -covermode=atomic ./pool/static_pool + go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/worker.out -covermode=atomic ./worker + go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/worker_stack.out -covermode=atomic ./worker_watcher + echo 'mode: atomic' > ./coverage-ci/summary.txt + tail -q -n +2 ./coverage-ci/*.out >> ./coverage-ci/summary.txt + +test: ## Run application tests + go test -v -race ./ipc/pipe + go test -v -race ./ipc/socket + go test -v -race ./pool/static_pool + go test -v -race ./worker + go test -v -race ./worker_watcher + go test -v -race -fuzz=FuzzStaticPoolEcho -fuzztime=30s -tags=debug ./pool/static_pool diff --git a/README.md b/README.md new file mode 100644 index 0000000..6b6c8e6 --- /dev/null +++ b/README.md @@ -0,0 +1,23 @@ +## RoadRunner Go SDK + +

+ + + + + + +

+ +

+ + + + + + + + +

+ +This repository contains RoadRunner Go SDK. This is fairly low-level API for creating processes (workers), worker pools powered with a different transports (IPC, sockets, etc). diff --git a/fsm/fsm.go b/fsm/fsm.go new file mode 100644 index 0000000..7ccff5c --- /dev/null +++ b/fsm/fsm.go @@ -0,0 +1,158 @@ +package fsm + +import ( + "sync/atomic" + + "github.com/roadrunner-server/errors" + "go.uber.org/zap" +) + +// NewFSM returns new FSM implementation based on initial state +func NewFSM(initialState int64, log *zap.Logger) *Fsm { + return &Fsm{ + log: log, + currentState: &initialState, + } +} + +// Fsm is general https://en.wikipedia.org/wiki/Finite-state_machine to transition between worker states +type Fsm struct { + log *zap.Logger + numExecs uint64 + // to be lightweight, use UnixNano + lastUsed uint64 + currentState *int64 +} + +// CurrentState (see interface) +func (s *Fsm) CurrentState() int64 { + return atomic.LoadInt64(s.currentState) +} + +func (s *Fsm) Compare(state int64) bool { + return atomic.LoadInt64(s.currentState) == state +} + +/* +Transition moves worker from one state to another +*/ +func (s *Fsm) Transition(to int64) { + err := s.recognizer(to) + if err != nil { + s.log.Debug("fsm transition error", zap.Error(err)) + return + } + + atomic.StoreInt64(s.currentState, to) +} + +// String returns current StateImpl as string. +func (s *Fsm) String() string { + switch atomic.LoadInt64(s.currentState) { + case StateInactive: + return "inactive" + case StateReady: + return "ready" + case StateWorking: + return "working" + case StateInvalid: + return "invalid" + case StateStopping: + return "stopping" + case StateStopped: + return "stopped" + case StateErrored: + return "errored" + case StateDestroyed: + return "destroyed" + case StateMaxJobsReached: + return "maxJobsReached" + case StateIdleTTLReached: + return "idleTTLReached" + case StateTTLReached: + return "ttlReached" + case StateMaxMemoryReached: + return "maxMemoryReached" + } + + return "undefined" +} + +// NumExecs returns number of registered WorkerProcess execs. +func (s *Fsm) NumExecs() uint64 { + return atomic.LoadUint64(&s.numExecs) +} + +// IsActive returns true if WorkerProcess not Inactive or Stopped +func (s *Fsm) IsActive() bool { + return atomic.LoadInt64(s.currentState) == StateWorking || + atomic.LoadInt64(s.currentState) == StateReady +} + +// RegisterExec register new execution atomically +func (s *Fsm) RegisterExec() { + atomic.AddUint64(&s.numExecs, 1) +} + +// SetLastUsed Update last used time +func (s *Fsm) SetLastUsed(lu uint64) { + atomic.StoreUint64(&s.lastUsed, lu) +} + +func (s *Fsm) LastUsed() uint64 { + return atomic.LoadUint64(&s.lastUsed) +} + +// Acceptors (also called detectors or recognizers) produce binary output, +// indicating whether or not the received input is accepted. +// Each event of an acceptor is either accepting or non accepting. +func (s *Fsm) recognizer(to int64) error { + const op = errors.Op("fsm_recognizer") + switch to { + // to + case StateInactive: + // from + // No-one can transition to Inactive + if atomic.LoadInt64(s.currentState) == StateDestroyed { + return errors.E(op, errors.Errorf("can't transition from state: %s", s.String())) + } + // to from StateWorking/StateInactive only + case StateReady: + // from + switch atomic.LoadInt64(s.currentState) { + case StateWorking, StateInactive: + return nil + } + + return errors.E(op, errors.Errorf("can't transition from state: %s", s.String())) + // to + case StateWorking: + // from + // StateWorking can be transitioned only from StateReady + if atomic.LoadInt64(s.currentState) == StateReady { + return nil + } + + return errors.E(op, errors.Errorf("can't transition from state: %s", s.String())) + // to + case + StateInvalid, + StateStopping, + StateStopped, + StateMaxJobsReached, + StateErrored, + StateIdleTTLReached, + StateTTLReached, + StateMaxMemoryReached, + StateExecTTLReached: + // from + if atomic.LoadInt64(s.currentState) == StateDestroyed { + return errors.E(op, errors.Errorf("can't transition from state: %s", s.String())) + } + // to + case StateDestroyed: + return nil + } + + return nil +} diff --git a/fsm/state.go b/fsm/state.go new file mode 100755 index 0000000..d5ea623 --- /dev/null +++ b/fsm/state.go @@ -0,0 +1,31 @@ +package fsm + +// All worker states +const ( + // StateInactive - no associated process + StateInactive int64 = iota + // StateReady - ready for job. + StateReady + // StateWorking - working on given payload. + StateWorking + // StateInvalid - indicates that WorkerProcess is being disabled and will be removed. + StateInvalid + // StateStopping - process is being softly stopped. + StateStopping + // StateStopped - process has been terminated. + StateStopped + // StateDestroyed State of worker, when no need to allocate new one + StateDestroyed + // StateMaxJobsReached State of worker, when it reached executions limit + StateMaxJobsReached + // StateErrored - error StateImpl (can't be used). + StateErrored + // StateIdleTTLReached - worker idle TTL was reached + StateIdleTTLReached + // StateTTLReached - worker TTL was reached + StateTTLReached + // StateMaxMemoryReached - worker max memory limit was reached + StateMaxMemoryReached + // StateExecTTLReached - worker execution TTL was reached + StateExecTTLReached +) diff --git a/fsm/state_test.go b/fsm/state_test.go new file mode 100755 index 0000000..66f051a --- /dev/null +++ b/fsm/state_test.go @@ -0,0 +1,32 @@ +package fsm + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "go.uber.org/zap" +) + +func Test_NewState(t *testing.T) { + log, err := zap.NewDevelopment() + assert.NoError(t, err) + st := NewFSM(StateErrored, log) + + assert.Equal(t, "errored", st.String()) + + assert.Equal(t, "inactive", NewFSM(StateInactive, log).String()) + assert.Equal(t, "ready", NewFSM(StateReady, log).String()) + assert.Equal(t, "working", NewFSM(StateWorking, log).String()) + assert.Equal(t, "stopped", NewFSM(StateStopped, log).String()) + assert.Equal(t, "undefined", NewFSM(1000, log).String()) +} + +func Test_IsActive(t *testing.T) { + log, err := zap.NewDevelopment() + assert.NoError(t, err) + assert.False(t, NewFSM(StateInactive, log).IsActive()) + assert.True(t, NewFSM(StateReady, log).IsActive()) + assert.True(t, NewFSM(StateWorking, log).IsActive()) + assert.False(t, NewFSM(StateStopped, log).IsActive()) + assert.False(t, NewFSM(StateErrored, log).IsActive()) +} diff --git a/githooks-installer.sh b/githooks-installer.sh new file mode 100644 index 0000000..b97d0a9 --- /dev/null +++ b/githooks-installer.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +set -e + +cp ./.githooks/pre-commit .git/hooks/pre-commit + +echo "DONE" \ No newline at end of file diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..b666dc0 --- /dev/null +++ b/go.mod @@ -0,0 +1,27 @@ +module github.com/roadrunner-server/pool + +go 1.22.4 + +require ( + github.com/goccy/go-json v0.10.3 + github.com/roadrunner-server/errors v1.4.0 + github.com/roadrunner-server/events v1.0.0 + github.com/roadrunner-server/goridge/v3 v3.8.2 + github.com/shirou/gopsutil v3.21.11+incompatible + github.com/stretchr/testify v1.9.0 + go.uber.org/zap v1.27.0 + golang.org/x/sync v0.7.0 +) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/go-ole/go-ole v1.3.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/tklauser/go-sysconf v0.3.14 // indirect + github.com/tklauser/numcpus v0.8.0 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect + go.uber.org/multierr v1.11.0 // indirect + golang.org/x/sys v0.21.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..3cc4d92 --- /dev/null +++ b/go.sum @@ -0,0 +1,43 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= +github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= +github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= +github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/roadrunner-server/errors v1.4.0 h1:Odjg3VZrj1q5Y8ILwoN+JgERyv0pkhrWPNOM4h68iQ8= +github.com/roadrunner-server/errors v1.4.0/go.mod h1:78PvraAFj+Sxy5nDmo0S+h6rEMLFIDszWZxA3B0sPAs= +github.com/roadrunner-server/events v1.0.0 h1:r+DM2mVJbcJSxj7AoESvolgUQQYNcEjzKPdgTXz3lPI= +github.com/roadrunner-server/events v1.0.0/go.mod h1:KMcez/tib0yky9TR/0Ag8SZcgAn3kjzXWciZL/n2Hu8= +github.com/roadrunner-server/goridge/v3 v3.8.2 h1:4TpIJAMylMIVTva/L/STB4ZvYNVoQ77+Syr6abxj95c= +github.com/roadrunner-server/goridge/v3 v3.8.2/go.mod h1:7IIDW50j1saxnOxktFeUPpkSIfyvM/dYopTrbGWXboA= +github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= +github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tklauser/go-sysconf v0.3.14 h1:g5vzr9iPFFz24v2KZXs/pvpvh8/V9Fw6vQK5ZZb78yU= +github.com/tklauser/go-sysconf v0.3.14/go.mod h1:1ym4lWMLUOhuBOPGtRcJm7tEGX4SCYNEEEtghGG/8uY= +github.com/tklauser/numcpus v0.8.0 h1:Mx4Wwe/FjZLeQsK/6kt2EOepwwSl7SmJrK5bV/dXYgY= +github.com/tklauser/numcpus v0.8.0/go.mod h1:ZJZlAY+dmR4eut8epnzf0u/VwodKmryxR8txiloSqBE= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/protocol.go b/internal/protocol.go new file mode 100755 index 0000000..3653f85 --- /dev/null +++ b/internal/protocol.go @@ -0,0 +1,94 @@ +package internal + +import ( + "os" + "sync" + + "github.com/goccy/go-json" + "github.com/roadrunner-server/errors" + "github.com/roadrunner-server/goridge/v3/pkg/frame" + "github.com/roadrunner-server/goridge/v3/pkg/relay" +) + +type StopCommand struct { + Stop bool `json:"stop"` +} + +type pidCommand struct { + Pid int `json:"pid"` +} + +var fPool = sync.Pool{New: func() any { + return frame.NewFrame() +}} + +func getFrame() *frame.Frame { + return fPool.Get().(*frame.Frame) +} + +func putFrame(f *frame.Frame) { + f.Reset() + fPool.Put(f) +} + +func SendControl(rl relay.Relay, payload any) error { + fr := getFrame() + defer putFrame(fr) + + fr.WriteVersion(fr.Header(), frame.Version1) + fr.WriteFlags(fr.Header(), frame.CONTROL, frame.CodecJSON) + + data, err := json.Marshal(payload) + if err != nil { + return errors.Errorf("invalid payload: %s", err) + } + + fr.WritePayloadLen(fr.Header(), uint32(len(data))) + fr.WritePayload(data) + fr.WriteCRC(fr.Header()) + + // we don't need a copy here, because frame copy the data before send + err = rl.Send(fr) + if err != nil { + return err + } + + return nil +} + +func Pid(rl relay.Relay) (int64, error) { + err := SendControl(rl, pidCommand{Pid: os.Getpid()}) + if err != nil { + return 0, err + } + + fr := getFrame() + defer putFrame(fr) + + err = rl.Receive(fr) + if err != nil { + return 0, err + } + + if fr == nil { + return 0, errors.Str("nil frame received") + } + + flags := fr.ReadFlags() + + if flags&frame.CONTROL == 0 { + return 0, errors.Str("unexpected response, header is missing, no CONTROL flag") + } + + link := &pidCommand{} + err = json.Unmarshal(fr.Payload(), link) + if err != nil { + return 0, err + } + + if link.Pid <= 0 { + return 0, errors.Str("pid should be greater than 0") + } + + return int64(link.Pid), nil +} diff --git a/ipc/pipe/pipe.go b/ipc/pipe/pipe.go new file mode 100755 index 0000000..bc47a7f --- /dev/null +++ b/ipc/pipe/pipe.go @@ -0,0 +1,146 @@ +package pipe + +import ( + "context" + "os/exec" + + "github.com/roadrunner-server/errors" + "github.com/roadrunner-server/goridge/v3/pkg/pipe" + "github.com/roadrunner-server/pool/fsm" + "github.com/roadrunner-server/pool/internal" + "github.com/roadrunner-server/pool/worker" + "go.uber.org/zap" +) + +// Factory connects to stack using standard +// streams (STDIN, STDOUT pipes). +type Factory struct { + log *zap.Logger +} + +// NewPipeFactory returns new factory instance and starts +// listening +func NewPipeFactory(log *zap.Logger) *Factory { + return &Factory{ + log: log, + } +} + +type sr struct { + w *worker.Process + err error +} + +// SpawnWorkerWithContext Creates a new Process and connects it to goridge relay, +// method Wait() must be handled on the level above. +func (f *Factory) SpawnWorkerWithContext(ctx context.Context, cmd *exec.Cmd, options ...worker.Options) (*worker.Process, error) { + spCh := make(chan sr) + go func() { + w, err := worker.InitBaseWorker(cmd, options...) + if err != nil { + select { + case spCh <- sr{ + w: nil, + err: err, + }: + return + default: + return + } + } + + in, err := cmd.StdoutPipe() + if err != nil { + select { + case spCh <- sr{ + w: nil, + err: err, + }: + return + default: + return + } + } + + out, err := cmd.StdinPipe() + if err != nil { + select { + case spCh <- sr{ + w: nil, + err: err, + }: + return + default: + return + } + } + + // Init new PIPE relay + relay := pipe.NewPipeRelay(in, out) + w.AttachRelay(relay) + + // Start the worker + err = w.Start() + if err != nil { + select { + case spCh <- sr{ + w: nil, + err: err, + }: + return + default: + return + } + } + + // used as a ping + _, err = internal.Pid(relay) + if err != nil { + go func() { + _ = w.Wait() + }() + _ = w.Kill() + select { + case spCh <- sr{ + w: nil, + err: err, + }: + return + default: + _ = w.Kill() + return + } + } + + // everything ok, set ready state + w.State().Transition(fsm.StateReady) + + select { + case + // return worker + spCh <- sr{ + w: w, + err: nil, + }: + return + default: + _ = w.Kill() + return + } + }() + + select { + case <-ctx.Done(): + return nil, errors.E(errors.TimeOut) + case res := <-spCh: + if res.err != nil { + return nil, res.err + } + return res.w, nil + } +} + +// Close the factory. +func (f *Factory) Close() error { + return nil +} diff --git a/ipc/pipe/pipe_spawn_test.go b/ipc/pipe/pipe_spawn_test.go new file mode 100644 index 0000000..1cd4d6f --- /dev/null +++ b/ipc/pipe/pipe_spawn_test.go @@ -0,0 +1,414 @@ +package pipe + +import ( + "context" + "os/exec" + "sync" + "testing" + "time" + + "github.com/roadrunner-server/errors" + "github.com/roadrunner-server/pool/fsm" + "github.com/roadrunner-server/pool/payload" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" +) + +var log = zap.NewNop() + +func Test_GetState2(t *testing.T) { + cmd := exec.Command("php", "../../tests/client.php", "echo", "pipes") + + w, err := NewPipeFactory(log).SpawnWorkerWithContext(context.Background(), cmd) + go func() { + assert.NoError(t, w.Wait()) + assert.Equal(t, fsm.StateStopped, w.State().CurrentState()) + }() + + assert.NoError(t, err) + assert.NotNil(t, w) + + assert.Equal(t, fsm.StateReady, w.State().CurrentState()) + assert.NoError(t, w.Stop()) +} + +func Test_Kill2(t *testing.T) { + cmd := exec.Command("php", "../../tests/client.php", "echo", "pipes") + + w, err := NewPipeFactory(log).SpawnWorkerWithContext(context.Background(), cmd) + wg := &sync.WaitGroup{} + wg.Add(1) + go func() { + defer wg.Done() + assert.Error(t, w.Wait()) + assert.Equal(t, fsm.StateErrored, w.State().CurrentState()) + }() + + assert.NoError(t, err) + assert.NotNil(t, w) + + assert.Equal(t, fsm.StateReady, w.State().CurrentState()) + err = w.Kill() + if err != nil { + t.Errorf("error killing the Process: error %v", err) + } + wg.Wait() +} + +func Test_Pipe_Start2(t *testing.T) { + cmd := exec.Command("php", "../../tests/client.php", "echo", "pipes") + + w, err := NewPipeFactory(log).SpawnWorkerWithContext(context.Background(), cmd) + assert.NoError(t, err) + assert.NotNil(t, w) + + go func() { + assert.NoError(t, w.Wait()) + }() + + assert.NoError(t, w.Stop()) +} + +func Test_Pipe_StartError2(t *testing.T) { + cmd := exec.Command("php", "../../tests/client.php", "echo", "pipes") + err := cmd.Start() + if err != nil { + t.Errorf("error running the command: error %v", err) + } + + w, err := NewPipeFactory(log).SpawnWorkerWithContext(context.Background(), cmd) + assert.Error(t, err) + assert.Nil(t, w) +} + +func Test_Pipe_PipeError3(t *testing.T) { + cmd := exec.Command("php", "../../tests/client.php", "echo", "pipes") + _, err := cmd.StdinPipe() + if err != nil { + t.Errorf("error creating the STDIN pipe: error %v", err) + } + + w, err := NewPipeFactory(log).SpawnWorkerWithContext(context.Background(), cmd) + assert.Error(t, err) + assert.Nil(t, w) +} + +func Test_Pipe_PipeError4(t *testing.T) { + cmd := exec.Command("php", "../../tests/client.php", "echo", "pipes") + _, err := cmd.StdinPipe() + if err != nil { + t.Errorf("error creating the STDIN pipe: error %v", err) + } + + w, err := NewPipeFactory(log).SpawnWorkerWithContext(context.Background(), cmd) + assert.Error(t, err) + assert.Nil(t, w) +} + +func Test_Pipe_Failboot2(t *testing.T) { + cmd := exec.Command("php", "../../tests/failboot.php") + w, err := NewPipeFactory(log).SpawnWorkerWithContext(context.Background(), cmd) + assert.Nil(t, w) + assert.Error(t, err) +} + +func Test_Pipe_Invalid2(t *testing.T) { + cmd := exec.Command("php", "../../tests/invalid.php") + w, err := NewPipeFactory(log).SpawnWorkerWithContext(context.Background(), cmd) + assert.Error(t, err) + assert.Nil(t, w) +} + +func Test_Pipe_Echo2(t *testing.T) { + cmd := exec.Command("php", "../../tests/client.php", "echo", "pipes") + w, err := NewPipeFactory(log).SpawnWorkerWithContext(context.Background(), cmd) + assert.NoError(t, err) + + res, err := w.Exec(context.Background(), &payload.Payload{Body: []byte("hello")}) + + assert.NoError(t, err) + assert.NotNil(t, res) + assert.NotNil(t, res.Body) + assert.Empty(t, res.Context) + + go func() { + if w.Wait() != nil { + t.Fail() + } + }() + + assert.Equal(t, "hello", res.String()) + err = w.Stop() + assert.NoError(t, err) +} + +func Test_Pipe_Broken2(t *testing.T) { + cmd := exec.Command("php", "../../tests/client.php", "broken", "pipes") + w, err := NewPipeFactory(log).SpawnWorkerWithContext(context.Background(), cmd) + assert.NoError(t, err) + require.NotNil(t, w) + + res, err := w.Exec(context.Background(), &payload.Payload{Body: []byte("hello")}) + assert.Error(t, err) + assert.Nil(t, res) + + time.Sleep(time.Second) + err = w.Stop() + assert.Error(t, err) +} + +func Benchmark_Pipe_SpawnWorker_Stop2(b *testing.B) { + f := NewPipeFactory(log) + for n := 0; n < b.N; n++ { + cmd := exec.Command("php", "../../tests/client.php", "echo", "pipes") + w, _ := f.SpawnWorkerWithContext(context.Background(), cmd) + go func() { + if w.Wait() != nil { + b.Fail() + } + }() + + err := w.Stop() + if err != nil { + b.Errorf("error stopping the worker: error %v", err) + } + } +} + +func Benchmark_Pipe_Worker_ExecEcho2(b *testing.B) { + cmd := exec.Command("php", "../../tests/client.php", "echo", "pipes") + + w, _ := NewPipeFactory(log).SpawnWorkerWithContext(context.Background(), cmd) + + b.ReportAllocs() + b.ResetTimer() + go func() { + err := w.Wait() + if err != nil { + b.Errorf("error waiting the worker: error %v", err) + } + }() + defer func() { + err := w.Stop() + if err != nil { + b.Errorf("error stopping the worker: error %v", err) + } + }() + + for n := 0; n < b.N; n++ { + if _, err := w.Exec(context.Background(), &payload.Payload{Body: []byte("hello")}); err != nil { + b.Fail() + } + } +} + +func Benchmark_Pipe_Worker_ExecEcho4(b *testing.B) { + cmd := exec.Command("php", "../../tests/client.php", "echo", "pipes") + w, err := NewPipeFactory(log).SpawnWorkerWithContext(context.Background(), cmd) + if err != nil { + b.Fatal(err) + } + + defer func() { + err = w.Stop() + if err != nil { + b.Errorf("error stopping the Process: error %v", err) + } + }() + + for n := 0; n < b.N; n++ { + if _, err := w.Exec(context.Background(), &payload.Payload{Body: []byte("hello")}); err != nil { + b.Fail() + } + } +} + +func Benchmark_Pipe_Worker_ExecEchoWithoutContext2(b *testing.B) { + cmd := exec.Command("php", "../../tests/client.php", "echo", "pipes") + w, err := NewPipeFactory(log).SpawnWorkerWithContext(context.Background(), cmd) + if err != nil { + b.Fatal(err) + } + + defer func() { + err = w.Stop() + if err != nil { + b.Errorf("error stopping the Process: error %v", err) + } + }() + + for n := 0; n < b.N; n++ { + if _, err := w.Exec(context.Background(), &payload.Payload{Body: []byte("hello")}); err != nil { + b.Fail() + } + } +} + +func Test_Echo2(t *testing.T) { + cmd := exec.Command("php", "../../tests/client.php", "echo", "pipes") + + w, err := NewPipeFactory(log).SpawnWorkerWithContext(context.Background(), cmd) + if err != nil { + t.Fatal(err) + } + + go func() { + assert.NoError(t, w.Wait()) + }() + defer func() { + err = w.Stop() + if err != nil { + t.Errorf("error stopping the Process: error %v", err) + } + }() + + res, err := w.Exec(context.Background(), &payload.Payload{Body: []byte("hello")}) + + assert.Nil(t, err) + assert.NotNil(t, res) + assert.NotNil(t, res.Body) + assert.Empty(t, res.Context) + + assert.Equal(t, "hello", res.String()) +} + +func Test_BadPayload2(t *testing.T) { + cmd := exec.Command("php", "../../tests/client.php", "echo", "pipes") + + w, _ := NewPipeFactory(log).SpawnWorkerWithContext(context.Background(), cmd) + + go func() { + assert.NoError(t, w.Wait()) + }() + defer func() { + err := w.Stop() + if err != nil { + t.Errorf("error stopping the Process: error %v", err) + } + }() + + res, err := w.Exec(context.Background(), &payload.Payload{}) + assert.NoError(t, err) + assert.NotNil(t, res) +} + +func Test_String2(t *testing.T) { + cmd := exec.Command("php", "../../tests/client.php", "echo", "pipes") + + w, _ := NewPipeFactory(log).SpawnWorkerWithContext(context.Background(), cmd) + go func() { + assert.NoError(t, w.Wait()) + }() + defer func() { + err := w.Stop() + if err != nil { + t.Errorf("error stopping the Process: error %v", err) + } + }() + + assert.Contains(t, w.String(), "php ../../tests/client.php echo pipes") + assert.Contains(t, w.String(), "ready") + assert.Contains(t, w.String(), "num_execs: 0") +} + +func Test_Echo_Slow2(t *testing.T) { + cmd := exec.Command("php", "../../tests/slow-client.php", "echo", "pipes", "10", "10") + + w, _ := NewPipeFactory(log).SpawnWorkerWithContext(context.Background(), cmd) + go func() { + assert.NoError(t, w.Wait()) + }() + defer func() { + err := w.Stop() + if err != nil { + t.Errorf("error stopping the Process: error %v", err) + } + }() + + res, err := w.Exec(context.Background(), &payload.Payload{Body: []byte("hello")}) + + assert.Nil(t, err) + assert.NotNil(t, res) + assert.NotNil(t, res.Body) + assert.Empty(t, res.Context) + + assert.Equal(t, "hello", res.String()) +} + +func Test_Broken2(t *testing.T) { + cmd := exec.Command("php", "../../tests/client.php", "broken", "pipes") + + w, err := NewPipeFactory(log).SpawnWorkerWithContext(context.Background(), cmd) + if err != nil { + t.Fatal(err) + } + + res, err := w.Exec(context.Background(), &payload.Payload{Body: []byte("hello")}) + assert.NotNil(t, err) + assert.Nil(t, res) + + time.Sleep(time.Second * 3) + assert.Error(t, w.Stop()) +} + +func Test_Error2(t *testing.T) { + cmd := exec.Command("php", "../../tests/client.php", "error", "pipes") + + w, _ := NewPipeFactory(log).SpawnWorkerWithContext(context.Background(), cmd) + go func() { + assert.NoError(t, w.Wait()) + }() + + defer func() { + err := w.Stop() + if err != nil { + t.Errorf("error stopping the Process: error %v", err) + } + }() + + res, err := w.Exec(context.Background(), &payload.Payload{Body: []byte("hello")}) + assert.NotNil(t, err) + assert.Nil(t, res) + + if errors.Is(errors.SoftJob, err) == false { + t.Fatal("error should be of type errors.ErrSoftJob") + } + assert.Contains(t, err.Error(), "hello") +} + +func Test_NumExecs2(t *testing.T) { + cmd := exec.Command("php", "../../tests/client.php", "echo", "pipes") + + w, _ := NewPipeFactory(log).SpawnWorkerWithContext(context.Background(), cmd) + go func() { + assert.NoError(t, w.Wait()) + }() + defer func() { + err := w.Stop() + if err != nil { + t.Errorf("error stopping the Process: error %v", err) + } + }() + + _, err := w.Exec(context.Background(), &payload.Payload{Body: []byte("hello")}) + if err != nil { + t.Errorf("fail to execute payload: error %v", err) + } + assert.Equal(t, uint64(1), w.State().NumExecs()) + w.State().Transition(fsm.StateReady) + + _, err = w.Exec(context.Background(), &payload.Payload{Body: []byte("hello")}) + if err != nil { + t.Errorf("fail to execute payload: error %v", err) + } + assert.Equal(t, uint64(2), w.State().NumExecs()) + w.State().Transition(fsm.StateReady) + + _, err = w.Exec(context.Background(), &payload.Payload{Body: []byte("hello")}) + if err != nil { + t.Errorf("fail to execute payload: error %v", err) + } + assert.Equal(t, uint64(3), w.State().NumExecs()) + w.State().Transition(fsm.StateReady) +} diff --git a/ipc/pipe/pipe_test.go b/ipc/pipe/pipe_test.go new file mode 100644 index 0000000..e635931 --- /dev/null +++ b/ipc/pipe/pipe_test.go @@ -0,0 +1,467 @@ +package pipe + +import ( + "context" + "os/exec" + "sync" + "testing" + "time" + + "github.com/roadrunner-server/errors" + "github.com/roadrunner-server/pool/fsm" + "github.com/roadrunner-server/pool/payload" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" +) + +func Test_GetState(t *testing.T) { + t.Parallel() + ctx := context.Background() + cmd := exec.Command("php", "../../tests/client.php", "echo", "pipes") + + w, err := NewPipeFactory(log).SpawnWorkerWithContext(ctx, cmd) + go func() { + assert.NoError(t, w.Wait()) + assert.Equal(t, fsm.StateStopped, w.State().CurrentState()) + }() + + assert.NoError(t, err) + assert.NotNil(t, w) + + assert.Equal(t, fsm.StateReady, w.State().CurrentState()) + err = w.Stop() + if err != nil { + t.Errorf("error stopping the Process: error %v", err) + } +} + +func Test_Kill(t *testing.T) { + t.Parallel() + ctx := context.Background() + cmd := exec.Command("php", "../../tests/client.php", "echo", "pipes") + + w, err := NewPipeFactory(log).SpawnWorkerWithContext(ctx, cmd) + wg := &sync.WaitGroup{} + wg.Add(1) + go func() { + defer wg.Done() + assert.Error(t, w.Wait()) + assert.Equal(t, fsm.StateErrored, w.State().CurrentState()) + }() + + assert.NoError(t, err) + assert.NotNil(t, w) + + assert.Equal(t, fsm.StateReady, w.State().CurrentState()) + err = w.Kill() + if err != nil { + t.Errorf("error killing the Process: error %v", err) + } + wg.Wait() +} + +func Test_Pipe_Start(t *testing.T) { + t.Parallel() + ctx := context.Background() + cmd := exec.Command("php", "../../tests/client.php", "echo", "pipes") + + w, err := NewPipeFactory(log).SpawnWorkerWithContext(ctx, cmd) + assert.NoError(t, err) + assert.NotNil(t, w) + + go func() { + assert.NoError(t, w.Wait()) + }() + + assert.NoError(t, w.Stop()) +} + +func Test_Pipe_StartError(t *testing.T) { + t.Parallel() + cmd := exec.Command("php", "../../tests/client.php", "echo", "pipes") + err := cmd.Start() + if err != nil { + t.Errorf("error running the command: error %v", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + w, err := NewPipeFactory(log).SpawnWorkerWithContext(ctx, cmd) + assert.Error(t, err) + assert.Nil(t, w) +} + +func Test_Pipe_PipeError(t *testing.T) { + t.Parallel() + cmd := exec.Command("php", "../../tests/client.php", "echo", "pipes") + _, err := cmd.StdinPipe() + if err != nil { + t.Errorf("error creating the STDIN pipe: error %v", err) + } + + ctx := context.Background() + w, err := NewPipeFactory(log).SpawnWorkerWithContext(ctx, cmd) + assert.Error(t, err) + assert.Nil(t, w) +} + +func Test_Pipe_PipeError2(t *testing.T) { + t.Parallel() + cmd := exec.Command("php", "../../tests/client.php", "echo", "pipes") + // error cause + _, err := cmd.StdinPipe() + if err != nil { + t.Errorf("error creating the STDIN pipe: error %v", err) + } + + ctx := context.Background() + w, err := NewPipeFactory(log).SpawnWorkerWithContext(ctx, cmd) + assert.Error(t, err) + assert.Nil(t, w) +} + +func Test_Pipe_Failboot(t *testing.T) { + cmd := exec.Command("php", "../../tests/failboot.php") + ctx := context.Background() + + w, err := NewPipeFactory(log).SpawnWorkerWithContext(ctx, cmd) + + assert.Nil(t, w) + assert.Error(t, err) +} + +func Test_Pipe_Invalid(t *testing.T) { + t.Parallel() + cmd := exec.Command("php", "../../tests/invalid.php") + ctx := context.Background() + w, err := NewPipeFactory(log).SpawnWorkerWithContext(ctx, cmd) + assert.Error(t, err) + assert.Nil(t, w) +} + +func Test_Pipe_Echo(t *testing.T) { + t.Parallel() + cmd := exec.Command("php", "../../tests/client.php", "echo", "pipes") + ctx := context.Background() + w, err := NewPipeFactory(log).SpawnWorkerWithContext(ctx, cmd) + if err != nil { + t.Fatal(err) + } + + defer func() { + err = w.Stop() + if err != nil { + t.Errorf("error stopping the Process: error %v", err) + } + }() + + res, err := w.Exec(context.Background(), &payload.Payload{Body: []byte("hello")}) + assert.NoError(t, err) + assert.NotNil(t, res) + assert.NotNil(t, res.Body) + assert.Empty(t, res.Context) + + go func() { + if w.Wait() != nil { + t.Fail() + } + }() + + assert.Equal(t, "hello", res.String()) +} + +func Test_Pipe_Broken(t *testing.T) { + t.Parallel() + cmd := exec.Command("php", "../../tests/client.php", "broken", "pipes") + ctx := context.Background() + w, err := NewPipeFactory(log).SpawnWorkerWithContext(ctx, cmd) + require.NoError(t, err) + require.NotNil(t, w) + + go func() { + errW := w.Wait() + require.Error(t, errW) + }() + + res, err := w.Exec(context.Background(), &payload.Payload{Body: []byte("hello")}) + assert.Error(t, err) + assert.Nil(t, res) + + time.Sleep(time.Second) + err = w.Stop() + assert.NoError(t, err) +} + +func Benchmark_Pipe_SpawnWorker_Stop(b *testing.B) { + f := NewPipeFactory(log) + for n := 0; n < b.N; n++ { + cmd := exec.Command("php", "../../tests/client.php", "echo", "pipes") + w, _ := f.SpawnWorkerWithContext(context.Background(), cmd) + go func() { + if w.Wait() != nil { + b.Fail() + } + }() + + err := w.Stop() + if err != nil { + b.Errorf("error stopping the worker: error %v", err) + } + } +} + +func Benchmark_Pipe_Worker_ExecEcho(b *testing.B) { + cmd := exec.Command("php", "../../tests/client.php", "echo", "pipes") + + w, _ := NewPipeFactory(log).SpawnWorkerWithContext(context.Background(), cmd) + + b.ReportAllocs() + b.ResetTimer() + go func() { + err := w.Wait() + if err != nil { + b.Errorf("error waiting the worker: error %v", err) + } + }() + defer func() { + err := w.Stop() + if err != nil { + b.Errorf("error stopping the worker: error %v", err) + } + }() + + for n := 0; n < b.N; n++ { + if _, err := w.Exec(context.Background(), &payload.Payload{Body: []byte("hello")}); err != nil { + b.Fail() + } + } +} + +func Benchmark_Pipe_Worker_ExecEchoWithoutContext(b *testing.B) { + cmd := exec.Command("php", "../../tests/client.php", "echo", "pipes") + ctx := context.Background() + w, err := NewPipeFactory(log).SpawnWorkerWithContext(ctx, cmd) + if err != nil { + b.Fatal(err) + } + + defer func() { + err = w.Stop() + if err != nil { + b.Errorf("error stopping the Process: error %v", err) + } + }() + + for n := 0; n < b.N; n++ { + if _, err := w.Exec(context.Background(), &payload.Payload{Body: []byte("hello")}); err != nil { + b.Fail() + } + } +} + +func Test_Echo(t *testing.T) { + t.Parallel() + ctx := context.Background() + cmd := exec.Command("php", "../../tests/client.php", "echo", "pipes") + + w, err := NewPipeFactory(log).SpawnWorkerWithContext(ctx, cmd) + if err != nil { + t.Fatal(err) + } + + go func() { + assert.NoError(t, w.Wait()) + }() + defer func() { + err = w.Stop() + if err != nil { + t.Errorf("error stopping the Process: error %v", err) + } + }() + + res, err := w.Exec(context.Background(), &payload.Payload{Body: []byte("hello")}) + + assert.Nil(t, err) + assert.NotNil(t, res) + assert.NotNil(t, res.Body) + assert.Empty(t, res.Context) + + assert.Equal(t, "hello", res.String()) +} + +func Test_BadPayload(t *testing.T) { + t.Parallel() + ctx := context.Background() + cmd := exec.Command("php", "../../tests/client.php", "echo", "pipes") + + w, _ := NewPipeFactory(log).SpawnWorkerWithContext(ctx, cmd) + + go func() { + assert.NoError(t, w.Wait()) + }() + defer func() { + err := w.Stop() + if err != nil { + t.Errorf("error stopping the Process: error %v", err) + } + }() + + res, err := w.Exec(context.Background(), &payload.Payload{}) + assert.NoError(t, err) + assert.NotNil(t, res) +} + +func Test_String(t *testing.T) { + t.Parallel() + ctx := context.Background() + cmd := exec.Command("php", "../../tests/client.php", "echo", "pipes") + + w, _ := NewPipeFactory(log).SpawnWorkerWithContext(ctx, cmd) + go func() { + assert.NoError(t, w.Wait()) + }() + defer func() { + err := w.Stop() + if err != nil { + t.Errorf("error stopping the Process: error %v", err) + } + }() + + assert.Contains(t, w.String(), "php ../../tests/client.php echo pipes") + assert.Contains(t, w.String(), "ready") + assert.Contains(t, w.String(), "num_execs: 0") +} + +func Test_Echo_Slow(t *testing.T) { + t.Parallel() + ctx := context.Background() + cmd := exec.Command("php", "../../tests/slow-client.php", "echo", "pipes", "10", "10") + + w, _ := NewPipeFactory(log).SpawnWorkerWithContext(ctx, cmd) + go func() { + assert.NoError(t, w.Wait()) + }() + defer func() { + err := w.Stop() + if err != nil { + t.Errorf("error stopping the Process: error %v", err) + } + }() + + res, err := w.Exec(context.Background(), &payload.Payload{Body: []byte("hello")}) + + assert.Nil(t, err) + assert.NotNil(t, res) + assert.NotNil(t, res.Body) + assert.Empty(t, res.Context) + + assert.Equal(t, "hello", res.String()) +} + +func Test_Broken(t *testing.T) { + ctx := context.Background() + cmd := exec.Command("php", "../../tests/client.php", "broken", "pipes") + + w, err := NewPipeFactory(log).SpawnWorkerWithContext(ctx, cmd) + if err != nil { + t.Fatal(err) + } + + res, err := w.Exec(context.Background(), &payload.Payload{Body: []byte("hello")}) + assert.NotNil(t, err) + assert.Nil(t, res) + + time.Sleep(time.Second * 3) + assert.Error(t, w.Stop()) +} + +func Test_Error(t *testing.T) { + t.Parallel() + ctx := context.Background() + cmd := exec.Command("php", "../../tests/client.php", "error", "pipes") + + w, _ := NewPipeFactory(log).SpawnWorkerWithContext(ctx, cmd) + go func() { + assert.NoError(t, w.Wait()) + }() + + defer func() { + err := w.Stop() + if err != nil { + t.Errorf("error stopping the Process: error %v", err) + } + }() + + res, err := w.Exec(context.Background(), &payload.Payload{Body: []byte("hello")}) + assert.NotNil(t, err) + assert.Nil(t, res) + + if errors.Is(errors.SoftJob, err) == false { + t.Fatal("error should be of type errors.ErrSoftJob") + } + assert.Contains(t, err.Error(), "hello") +} + +func Test_NumExecs(t *testing.T) { + t.Parallel() + ctx := context.Background() + cmd := exec.Command("php", "../../tests/client.php", "echo", "pipes") + + w, _ := NewPipeFactory(log).SpawnWorkerWithContext(ctx, cmd) + go func() { + assert.NoError(t, w.Wait()) + }() + defer func() { + err := w.Stop() + if err != nil { + t.Errorf("error stopping the Process: error %v", err) + } + }() + + _, err := w.Exec(context.Background(), &payload.Payload{Body: []byte("hello")}) + if err != nil { + t.Errorf("fail to execute payload: error %v", err) + } + w.State().Transition(fsm.StateReady) + assert.Equal(t, uint64(1), w.State().NumExecs()) + + _, err = w.Exec(context.Background(), &payload.Payload{Body: []byte("hello")}) + if err != nil { + t.Errorf("fail to execute payload: error %v", err) + } + assert.Equal(t, uint64(2), w.State().NumExecs()) + w.State().Transition(fsm.StateReady) + + _, err = w.Exec(context.Background(), &payload.Payload{Body: []byte("hello")}) + if err != nil { + t.Errorf("fail to execute payload: error %v", err) + } + assert.Equal(t, uint64(3), w.State().NumExecs()) + w.State().Transition(fsm.StateReady) +} +func Benchmark_WorkerPipeTTL(b *testing.B) { + cmd := exec.Command("php", "../../tests/client.php", "echo", "pipes") + ctx := context.Background() + + log, _ = zap.NewDevelopment() + w, err := NewPipeFactory(log).SpawnWorkerWithContext(ctx, cmd) + require.NoError(b, err) + + go func() { + _ = w.Wait() + }() + + b.ResetTimer() + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + res, err := w.Exec(ctx, &payload.Payload{Body: []byte("hello")}) + assert.NoError(b, err) + assert.NotNil(b, res) + } + + b.Cleanup(func() { + assert.NoError(b, w.Stop()) + }) +} diff --git a/ipc/socket/socket.go b/ipc/socket/socket.go new file mode 100755 index 0000000..82ee7d2 --- /dev/null +++ b/ipc/socket/socket.go @@ -0,0 +1,193 @@ +package socket + +import ( + "context" + stderr "errors" + "net" + "os/exec" + "sync" + "time" + + "github.com/roadrunner-server/errors" + "github.com/roadrunner-server/goridge/v3/pkg/relay" + "github.com/roadrunner-server/goridge/v3/pkg/socket" + "github.com/roadrunner-server/pool/fsm" + "github.com/roadrunner-server/pool/internal" + "github.com/roadrunner-server/pool/worker" + "github.com/shirou/gopsutil/process" + "go.uber.org/zap" + + "golang.org/x/sync/errgroup" +) + +// Factory connects to external stack using socket server. +type Factory struct { + // listens for incoming connections from underlying processes + ls net.Listener + // sockets which are waiting for process association + relays sync.Map + log *zap.Logger +} + +// NewSocketServer returns Factory attached to a given socket listener. +func NewSocketServer(ls net.Listener, log *zap.Logger) *Factory { + f := &Factory{ + ls: ls, + log: log, + } + + // Be careful + // https://github.com/go101/go101/wiki/About-memory-ordering-guarantees-made-by-atomic-operations-in-Go + // https://github.com/golang/go/issues/5045 + go func() { + err := f.listen() + // there is no logger here, use fmt + if err != nil { + var opErr *net.OpError + if stderr.As(err, &opErr) { + if opErr.Err.Error() == "use of closed network connection" { + return + } + } + + log.Warn("socket server listen", zap.Error(err)) + } + }() + + return f +} + +// blocking operation, returns an error +func (f *Factory) listen() error { + errGr := &errgroup.Group{} + errGr.Go(func() error { + for { + conn, err := f.ls.Accept() + if err != nil { + return err + } + + rl := socket.NewSocketRelay(conn) + pid, err := internal.Pid(rl) + if err != nil { + return err + } + + f.attachRelayToPid(pid, rl) + } + }) + + return errGr.Wait() +} + +type socketSpawn struct { + w *worker.Process + err error +} + +// SpawnWorkerWithContext Creates a Process and connects it to the appropriate relay or return an error +func (f *Factory) SpawnWorkerWithContext(ctx context.Context, cmd *exec.Cmd, options ...worker.Options) (*worker.Process, error) { + c := make(chan socketSpawn) + go func() { + w, err := worker.InitBaseWorker(cmd, options...) + if err != nil { + select { + case c <- socketSpawn{ + w: nil, + err: err, + }: + return + default: + return + } + } + + err = w.Start() + if err != nil { + select { + case c <- socketSpawn{ + w: nil, + err: err, + }: + return + default: + return + } + } + + rl, err := f.findRelayWithContext(ctx, w) + if err != nil { + _ = w.Kill() + select { + // try to write a result + case c <- socketSpawn{ + w: nil, + err: err, + }: + return + // if no receivers - return + default: + return + } + } + + w.AttachRelay(rl) + w.State().Transition(fsm.StateReady) + + select { + case c <- socketSpawn{ + w: w, + err: nil, + }: + return + default: + _ = w.Kill() + return + } + }() + + select { + case <-ctx.Done(): + return nil, errors.E(errors.TimeOut) + case res := <-c: + if res.err != nil { + return nil, res.err + } + + return res.w, nil + } +} + +// Close socket factory and underlying socket connection. +func (f *Factory) Close() error { + return f.ls.Close() +} + +// waits for Process to connect over socket and returns associated relay of timeout +func (f *Factory) findRelayWithContext(ctx context.Context, w *worker.Process) (*socket.Relay, error) { + ticker := time.NewTicker(time.Millisecond * 10) + for { + select { + case <-ctx.Done(): + return nil, errors.E(errors.Op("findRelayWithContext"), errors.TimeOut) + case <-ticker.C: + // check for the process exists + _, err := process.NewProcess(int32(w.Pid())) + if err != nil { + return nil, err + } + default: + rl, ok := f.relays.LoadAndDelete(w.Pid()) + if !ok { + continue + } + + return rl.(*socket.Relay), nil + } + } +} + +// chan to store relay associated with specific pid +func (f *Factory) attachRelayToPid(pid int64, relay relay.Relay) { + f.relays.Store(pid, relay) +} diff --git a/ipc/socket/socket_spawn_test.go b/ipc/socket/socket_spawn_test.go new file mode 100644 index 0000000..ebd7983 --- /dev/null +++ b/ipc/socket/socket_spawn_test.go @@ -0,0 +1,521 @@ +package socket + +import ( + "context" + "net" + "os/exec" + "sync" + "testing" + "time" + + "github.com/roadrunner-server/pool/payload" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" +) + +var log = zap.NewNop() + +func Test_Tcp_Start2(t *testing.T) { + ls, err := net.Listen("tcp", "127.0.0.1:9007") + if assert.NoError(t, err) { + defer func() { + errC := ls.Close() + require.NoError(t, errC) + }() + } else { + t.Skip("socket is busy") + } + + cmd := exec.Command("php", "../../tests/client.php", "echo", "tcp") + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + + w, err := NewSocketServer(ls, log).SpawnWorkerWithContext(ctx, cmd) + assert.NoError(t, err) + assert.NotNil(t, w) + + go func() { + assert.NoError(t, w.Wait()) + }() + + err = w.Stop() + if err != nil { + t.Errorf("error stopping the Process: error %v", err) + } +} + +func Test_Tcp_StartCloseFactory2(t *testing.T) { + ls, err := net.Listen("tcp", "127.0.0.1:9007") + if assert.NoError(t, err) { + } else { + t.Skip("socket is busy") + } + + cmd := exec.Command("php", "../../tests/client.php", "echo", "tcp") + f := NewSocketServer(ls, log) + defer func() { + err = ls.Close() + if err != nil { + t.Errorf("error closing the listener: error %v", err) + } + }() + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + + w, err := f.SpawnWorkerWithContext(ctx, cmd) + assert.NoError(t, err) + assert.NotNil(t, w) + + go func() { + require.NoError(t, w.Wait()) + }() + + err = w.Stop() + if err != nil { + t.Errorf("error stopping the Process: error %v", err) + } +} + +func Test_Tcp_StartError2(t *testing.T) { + ls, err := net.Listen("tcp", "127.0.0.1:9007") + if assert.NoError(t, err) { + defer func() { + errC := ls.Close() + require.NoError(t, errC) + }() + } else { + t.Skip("socket is busy") + } + + cmd := exec.Command("php", "../../tests/client.php", "echo", "pipes") + err = cmd.Start() + if err != nil { + t.Errorf("error executing the command: error %v", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + + w, err := NewSocketServer(ls, log).SpawnWorkerWithContext(ctx, cmd) + assert.Error(t, err) + assert.Nil(t, w) +} + +func Test_Tcp_Failboot2(t *testing.T) { + ls, err := net.Listen("tcp", "127.0.0.1:9007") + if assert.NoError(t, err) { + defer func() { + err3 := ls.Close() + if err3 != nil { + t.Errorf("error closing the listener: error %v", err3) + } + }() + } else { + t.Skip("socket is busy") + } + + cmd := exec.Command("php", "../../tests/failboot.php") + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + + w, err2 := NewSocketServer(ls, log).SpawnWorkerWithContext(ctx, cmd) + assert.Nil(t, w) + assert.Error(t, err2) +} + +func Test_Tcp_Invalid2(t *testing.T) { + ls, err := net.Listen("tcp", "127.0.0.1:9007") + if assert.NoError(t, err) { + defer func() { + errC := ls.Close() + require.NoError(t, errC) + }() + } else { + t.Skip("socket is busy") + } + + cmd := exec.Command("php", "../../tests/invalid.php") + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*15) + defer cancel() + + w, err := NewSocketServer(ls, log).SpawnWorkerWithContext(ctx, cmd) + assert.Error(t, err) + assert.Nil(t, w) +} + +func Test_Tcp_Broken2(t *testing.T) { + ls, err := net.Listen("tcp", "127.0.0.1:9007") + if assert.NoError(t, err) { + defer func() { + errC := ls.Close() + require.NoError(t, errC) + }() + } else { + t.Skip("socket is busy") + } + + cmd := exec.Command("php", "../../tests/client.php", "broken", "tcp") + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*15) + defer cancel() + + w, err := NewSocketServer(ls, log).SpawnWorkerWithContext(ctx, cmd) + if err != nil { + t.Fatal(err) + } + wg := sync.WaitGroup{} + wg.Add(1) + go func() { + defer wg.Done() + errW := w.Wait() + assert.Error(t, errW) + }() + + res, err := w.Exec(context.Background(), &payload.Payload{Body: []byte("hello")}) + assert.Error(t, err) + assert.Nil(t, res) + wg.Wait() + + time.Sleep(time.Second) + err2 := w.Stop() + // write tcp 127.0.0.1:9007->127.0.0.1:34204: use of closed network connection + // but process exited + assert.NoError(t, err2) +} + +func Test_Tcp_Echo2(t *testing.T) { + ls, err := net.Listen("tcp", "127.0.0.1:9007") + if assert.NoError(t, err) { + defer func() { + errC := ls.Close() + require.NoError(t, errC) + }() + } else { + t.Skip("socket is busy") + } + + cmd := exec.Command("php", "../../tests/client.php", "echo", "tcp") + + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + w, _ := NewSocketServer(ls, log).SpawnWorkerWithContext(ctx, cmd) + go func() { + assert.NoError(t, w.Wait()) + }() + defer func() { + err = w.Stop() + if err != nil { + t.Errorf("error stopping the Process: error %v", err) + } + }() + + res, err := w.Exec(context.Background(), &payload.Payload{Body: []byte("hello")}) + + assert.NoError(t, err) + assert.NotNil(t, res) + assert.NotNil(t, res.Body) + assert.Empty(t, res.Context) + + assert.Equal(t, "hello", res.String()) +} + +func Test_Unix_Start2(t *testing.T) { + ls, err := net.Listen("unix", "sock.unix") + if assert.NoError(t, err) { + defer func() { + errC := ls.Close() + require.NoError(t, errC) + }() + } else { + t.Skip("socket is busy") + } + + cmd := exec.Command("php", "../../tests/client.php", "echo", "unix") + + w, err := NewSocketServer(ls, log).SpawnWorkerWithContext(context.Background(), cmd) + assert.NoError(t, err) + assert.NotNil(t, w) + + go func() { + assert.NoError(t, w.Wait()) + }() + + err = w.Stop() + if err != nil { + t.Errorf("error stopping the Process: error %v", err) + } +} + +func Test_Unix_Failboot2(t *testing.T) { + ls, err := net.Listen("unix", "sock.unix") + if assert.NoError(t, err) { + defer func() { + errC := ls.Close() + require.NoError(t, errC) + }() + } else { + t.Skip("socket is busy") + } + + cmd := exec.Command("php", "../../tests/failboot.php") + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*15) + defer cancel() + + w, err := NewSocketServer(ls, log).SpawnWorkerWithContext(ctx, cmd) + assert.Nil(t, w) + assert.Error(t, err) +} + +func Test_Unix_Timeout2(t *testing.T) { + ls, err := net.Listen("unix", "sock.unix") + if assert.NoError(t, err) { + defer func() { + errC := ls.Close() + require.NoError(t, errC) + }() + } else { + t.Skip("socket is busy") + } + + cmd := exec.Command("php", "../../tests/slow-client.php", "echo", "unix", "200", "0") + + ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*100) + defer cancel() + + w, err := NewSocketServer(ls, log).SpawnWorkerWithContext(ctx, cmd) + + assert.Nil(t, w) + assert.Error(t, err) + assert.Contains(t, err.Error(), "Timeout") +} + +func Test_Unix_Invalid2(t *testing.T) { + ls, err := net.Listen("unix", "sock.unix") + if assert.NoError(t, err) { + defer func() { + errC := ls.Close() + require.NoError(t, errC) + }() + } else { + t.Skip("socket is busy") + } + + cmd := exec.Command("php", "../../tests/invalid.php") + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*15) + defer cancel() + + w, err := NewSocketServer(ls, log).SpawnWorkerWithContext(ctx, cmd) + assert.Error(t, err) + assert.Nil(t, w) +} + +func Test_Unix_Broken2(t *testing.T) { + ls, err := net.Listen("unix", "sock.unix") + if assert.NoError(t, err) { + defer func() { + errC := ls.Close() + require.NoError(t, errC) + }() + } else { + t.Skip("socket is busy") + } + + cmd := exec.Command("php", "../../tests/client.php", "broken", "unix") + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*15) + defer cancel() + + w, err := NewSocketServer(ls, log).SpawnWorkerWithContext(ctx, cmd) + if err != nil { + t.Fatal(err) + } + wg := &sync.WaitGroup{} + wg.Add(1) + go func() { + defer wg.Done() + errW := w.Wait() + assert.Error(t, errW) + }() + + res, err := w.Exec(context.Background(), &payload.Payload{Body: []byte("hello")}) + + assert.Error(t, err) + assert.Nil(t, res) + wg.Wait() + + time.Sleep(time.Second) + err = w.Stop() + assert.NoError(t, err) +} + +func Test_Unix_Echo2(t *testing.T) { + ls, err := net.Listen("unix", "sock.unix") + if assert.NoError(t, err) { + defer func() { + errC := ls.Close() + require.NoError(t, errC) + }() + } else { + t.Skip("socket is busy") + } + + cmd := exec.Command("php", "../../tests/client.php", "echo", "unix") + + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + w, err := NewSocketServer(ls, log).SpawnWorkerWithContext(ctx, cmd) + if err != nil { + t.Fatal(err) + } + go func() { + assert.NoError(t, w.Wait()) + }() + defer func() { + err = w.Stop() + if err != nil { + t.Errorf("error stopping the Process: error %v", err) + } + }() + + res, err := w.Exec(context.Background(), &payload.Payload{Body: []byte("hello")}) + + assert.NoError(t, err) + assert.NotNil(t, res) + assert.NotNil(t, res.Body) + assert.Empty(t, res.Context) + + assert.Equal(t, "hello", res.String()) +} + +func Benchmark_Tcp_SpawnWorker_Stop2(b *testing.B) { + ls, err := net.Listen("unix", "sock.unix") + if assert.NoError(b, err) { + defer func() { + errC := ls.Close() + require.NoError(b, errC) + }() + } else { + b.Skip("socket is busy") + } + + f := NewSocketServer(ls, log) + for n := 0; n < b.N; n++ { + cmd := exec.Command("php", "../../tests/client.php", "echo", "tcp") + + w, err := f.SpawnWorkerWithContext(context.Background(), cmd) + if err != nil { + b.Fatal(err) + } + go func() { + assert.NoError(b, w.Wait()) + }() + + err = w.Stop() + if err != nil { + b.Errorf("error stopping the Process: error %v", err) + } + } +} + +func Benchmark_Tcp_Worker_ExecEcho2(b *testing.B) { + ls, err := net.Listen("unix", "sock.unix") + if assert.NoError(b, err) { + defer func() { + errC := ls.Close() + require.NoError(b, errC) + }() + } else { + b.Skip("socket is busy") + } + + cmd := exec.Command("php", "../../tests/client.php", "echo", "tcp") + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*15) + defer cancel() + + w, err := NewSocketServer(ls, log).SpawnWorkerWithContext(ctx, cmd) + if err != nil { + b.Fatal(err) + } + defer func() { + err = w.Stop() + if err != nil { + b.Errorf("error stopping the Process: error %v", err) + } + }() + + for n := 0; n < b.N; n++ { + if _, err := w.Exec(context.Background(), &payload.Payload{Body: []byte("hello")}); err != nil { + b.Fail() + } + } +} + +func Benchmark_Unix_SpawnWorker_Stop2(b *testing.B) { + ls, err := net.Listen("unix", "sock.unix") + if err == nil { + defer func() { + errC := ls.Close() + require.NoError(b, errC) + }() + } else { + b.Skip("socket is busy") + } + + f := NewSocketServer(ls, log) + for n := 0; n < b.N; n++ { + cmd := exec.Command("php", "../../tests/client.php", "echo", "unix") + + w, err := f.SpawnWorkerWithContext(context.Background(), cmd) + if err != nil { + b.Fatal(err) + } + err = w.Stop() + if err != nil { + b.Errorf("error stopping the Process: error %v", err) + } + } +} + +func Benchmark_Unix_Worker_ExecEcho2(b *testing.B) { + ls, err := net.Listen("unix", "sock.unix") + if err == nil { + defer func() { + errC := ls.Close() + require.NoError(b, errC) + }() + } else { + b.Skip("socket is busy") + } + + cmd := exec.Command("php", "../../tests/client.php", "echo", "unix") + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*15) + defer cancel() + + w, err := NewSocketServer(ls, log).SpawnWorkerWithContext(ctx, cmd) + if err != nil { + b.Fatal(err) + } + defer func() { + err = w.Stop() + if err != nil { + b.Errorf("error stopping the Process: error %v", err) + } + }() + + for n := 0; n < b.N; n++ { + if _, err := w.Exec(context.Background(), &payload.Payload{Body: []byte("hello")}); err != nil { + b.Fail() + } + } +} diff --git a/ipc/socket/socket_test.go b/ipc/socket/socket_test.go new file mode 100755 index 0000000..f63ccdc --- /dev/null +++ b/ipc/socket/socket_test.go @@ -0,0 +1,567 @@ +package socket + +import ( + "context" + "net" + "os/exec" + "sync" + "testing" + "time" + + "github.com/roadrunner-server/errors" + "github.com/roadrunner-server/pool/payload" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func Test_Tcp_Start(t *testing.T) { + ctx := context.Background() + time.Sleep(time.Millisecond * 10) // to ensure free socket + + ls, err := net.Listen("tcp", "127.0.0.1:9007") + if assert.NoError(t, err) { + defer func() { + err = ls.Close() + if err != nil { + t.Errorf("error closing the listener: error %v", err) + } + }() + } else { + t.Skip("socket is busy") + } + + cmd := exec.Command("php", "../../tests/client.php", "echo", "tcp") + + w, err := NewSocketServer(ls, log).SpawnWorkerWithContext(ctx, cmd) + assert.NoError(t, err) + assert.NotNil(t, w) + + go func() { + assert.NoError(t, w.Wait()) + }() + + err = w.Stop() + if err != nil { + t.Errorf("error stopping the Process: error %v", err) + } +} + +func Test_Tcp_StartCloseFactory(t *testing.T) { + time.Sleep(time.Millisecond * 10) // to ensure free socket + ctx := context.Background() + ls, err := net.Listen("tcp", "127.0.0.1:9007") + if assert.NoError(t, err) { + } else { + t.Skip("socket is busy") + } + + cmd := exec.Command("php", "../../tests/client.php", "echo", "tcp") + f := NewSocketServer(ls, log) + defer func() { + err = ls.Close() + if err != nil { + t.Errorf("error closing the listener: error %v", err) + } + }() + + w, err := f.SpawnWorkerWithContext(ctx, cmd) + assert.NoError(t, err) + assert.NotNil(t, w) + + go func() { + require.NoError(t, w.Wait()) + }() + + err = w.Stop() + if err != nil { + t.Errorf("error stopping the Process: error %v", err) + } +} + +func Test_Tcp_StartError(t *testing.T) { + time.Sleep(time.Millisecond * 10) // to ensure free socket + ctx, cancel := context.WithTimeout(context.Background(), time.Second*15) + defer cancel() + + ls, err := net.Listen("tcp", "127.0.0.1:9007") + if assert.NoError(t, err) { + defer func() { + err = ls.Close() + if err != nil { + t.Errorf("error closing the listener: error %v", err) + } + }() + } else { + t.Skip("socket is busy") + } + + cmd := exec.Command("php", "../../tests/client.php", "echo", "pipes") + err = cmd.Start() + if err != nil { + t.Errorf("error executing the command: error %v", err) + } + + serv := NewSocketServer(ls, log) + time.Sleep(time.Second * 2) + w, err := serv.SpawnWorkerWithContext(ctx, cmd) + assert.Error(t, err) + assert.Nil(t, w) +} + +func Test_Tcp_Failboot(t *testing.T) { + time.Sleep(time.Millisecond * 10) // to ensure free socket + ctx, cancel := context.WithTimeout(context.Background(), time.Second*20) + defer cancel() + + ls, err := net.Listen("tcp", "127.0.0.1:9007") + if assert.NoError(t, err) { + defer func() { + err3 := ls.Close() + if err3 != nil { + t.Errorf("error closing the listener: error %v", err3) + } + }() + } else { + t.Skip("socket is busy") + } + + cmd := exec.Command("php", "../../tests/failboot.php") + + w, err2 := NewSocketServer(ls, log).SpawnWorkerWithContext(ctx, cmd) + assert.Nil(t, w) + assert.Error(t, err2) +} + +func Test_Tcp_Timeout(t *testing.T) { + time.Sleep(time.Millisecond * 10) // to ensure free socket + ctx, cancel := context.WithTimeout(context.Background(), time.Microsecond) + defer cancel() + + ls, err := net.Listen("tcp", "127.0.0.1:9007") + if assert.NoError(t, err) { + defer func() { + err = ls.Close() + if err != nil { + t.Errorf("error closing the listener: error %v", err) + } + }() + } else { + t.Skip("socket is busy") + } + + cmd := exec.Command("php", "../../tests/slow-client.php", "echo", "tcp", "200", "0") + + w, err := NewSocketServer(ls, log).SpawnWorkerWithContext(ctx, cmd) + assert.Nil(t, w) + assert.Error(t, err) + assert.True(t, errors.Is(errors.TimeOut, err)) +} + +func Test_Tcp_Invalid(t *testing.T) { + time.Sleep(time.Millisecond * 10) // to ensure free socket + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + ls, err := net.Listen("tcp", "127.0.0.1:9007") + if assert.NoError(t, err) { + defer func() { + err = ls.Close() + if err != nil { + t.Errorf("error closing the listener: error %v", err) + } + }() + } else { + t.Skip("socket is busy") + } + + cmd := exec.Command("php", "../../tests/invalid.php") + + w, err := NewSocketServer(ls, log).SpawnWorkerWithContext(ctx, cmd) + assert.Error(t, err) + assert.Nil(t, w) +} + +func Test_Tcp_Broken(t *testing.T) { + time.Sleep(time.Millisecond * 10) // to ensure free socket + ctx := context.Background() + ls, err := net.Listen("tcp", "127.0.0.1:9007") + if assert.NoError(t, err) { + defer func() { + errC := ls.Close() + if errC != nil { + t.Errorf("error closing the listener: error %v", err) + } + }() + } else { + t.Skip("socket is busy") + } + + cmd := exec.Command("php", "../../tests/client.php", "broken", "tcp") + + w, err := NewSocketServer(ls, log).SpawnWorkerWithContext(ctx, cmd) + if err != nil { + t.Fatal(err) + } + wg := sync.WaitGroup{} + wg.Add(1) + go func() { + defer wg.Done() + errW := w.Wait() + assert.Error(t, errW) + }() + + res, err := w.Exec(context.Background(), &payload.Payload{Body: []byte("hello")}) + assert.Error(t, err) + assert.Nil(t, res) + wg.Wait() + + time.Sleep(time.Second) + err2 := w.Stop() + // write tcp 127.0.0.1:9007->127.0.0.1:34204: use of closed network connection + // but process is stopped + assert.NoError(t, err2) +} + +func Test_Tcp_Echo(t *testing.T) { + time.Sleep(time.Millisecond * 10) // to ensure free socket + ctx := context.Background() + ls, err := net.Listen("tcp", "127.0.0.1:9007") + if assert.NoError(t, err) { + defer func() { + err = ls.Close() + if err != nil { + t.Errorf("error closing the listener: error %v", err) + } + }() + } else { + t.Skip("socket is busy") + } + + cmd := exec.Command("php", "../../tests/client.php", "echo", "tcp") + + w, _ := NewSocketServer(ls, log).SpawnWorkerWithContext(ctx, cmd) + go func() { + assert.NoError(t, w.Wait()) + }() + defer func() { + err = w.Stop() + if err != nil { + t.Errorf("error stopping the Process: error %v", err) + } + }() + + res, err := w.Exec(context.Background(), &payload.Payload{Body: []byte("hello")}) + + assert.NoError(t, err) + assert.NotNil(t, res) + assert.NotNil(t, res.Body) + assert.Empty(t, res.Context) + + assert.Equal(t, "hello", res.String()) +} + +func Test_Unix_Start(t *testing.T) { + ctx := context.Background() + ls, err := net.Listen("unix", "sock.unix") + if err == nil { + defer func() { + err = ls.Close() + if err != nil { + t.Errorf("error closing the listener: error %v", err) + } + }() + } else { + t.Skip("socket is busy") + } + + cmd := exec.Command("php", "../../tests/client.php", "echo", "unix") + + w, err := NewSocketServer(ls, log).SpawnWorkerWithContext(ctx, cmd) + assert.NoError(t, err) + assert.NotNil(t, w) + + go func() { + assert.NoError(t, w.Wait()) + }() + + err = w.Stop() + if err != nil { + t.Errorf("error stopping the Process: error %v", err) + } +} + +func Test_Unix_Failboot(t *testing.T) { + ls, err := net.Listen("unix", "sock.unix") + if err == nil { + defer func() { + err = ls.Close() + if err != nil { + t.Errorf("error closing the listener: error %v", err) + } + }() + } else { + t.Skip("socket is busy") + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + + cmd := exec.Command("php", "../../tests/failboot.php") + + w, err := NewSocketServer(ls, log).SpawnWorkerWithContext(ctx, cmd) + assert.Nil(t, w) + assert.Error(t, err) +} + +func Test_Unix_Timeout(t *testing.T) { + ls, err := net.Listen("unix", "sock.unix") + if err == nil { + defer func() { + err = ls.Close() + if err != nil { + t.Errorf("error closing the listener: error %v", err) + } + }() + } else { + t.Skip("socket is busy") + } + + cmd := exec.Command("php", "../../tests/slow-client.php", "echo", "unix", "200", "0") + ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*100) + defer cancel() + + w, err := NewSocketServer(ls, log).SpawnWorkerWithContext(ctx, cmd) + assert.Nil(t, w) + assert.Error(t, err) + assert.True(t, errors.Is(errors.TimeOut, err)) +} + +func Test_Unix_Invalid(t *testing.T) { + ls, err := net.Listen("unix", "sock.unix") + if err == nil { + defer func() { + err = ls.Close() + if err != nil { + t.Errorf("error closing the listener: error %v", err) + } + }() + } else { + t.Skip("socket is busy") + } + + cmd := exec.Command("php", "../../tests/invalid.php") + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + + w, err := NewSocketServer(ls, log).SpawnWorkerWithContext(ctx, cmd) + assert.Error(t, err) + assert.Nil(t, w) +} + +func Test_Unix_Broken(t *testing.T) { + ctx := context.Background() + ls, err := net.Listen("unix", "sock.unix") + if err == nil { + defer func() { + errC := ls.Close() + if errC != nil { + t.Errorf("error closing the listener: error %v", err) + } + }() + } else { + t.Skip("socket is busy") + } + + cmd := exec.Command("php", "../../tests/client.php", "broken", "unix") + w, err := NewSocketServer(ls, log).SpawnWorkerWithContext(ctx, cmd) + if err != nil { + t.Fatal(err) + } + wg := &sync.WaitGroup{} + wg.Add(1) + go func() { + defer wg.Done() + errW := w.Wait() + assert.Error(t, errW) + }() + + res, err := w.Exec(context.Background(), &payload.Payload{Body: []byte("hello")}) + + assert.Error(t, err) + assert.Nil(t, res) + + time.Sleep(time.Second) + err = w.Stop() + assert.NoError(t, err) + + wg.Wait() +} + +func Test_Unix_Echo(t *testing.T) { + ctx := context.Background() + ls, err := net.Listen("unix", "sock.unix") + if err == nil { + defer func() { + err = ls.Close() + if err != nil { + t.Errorf("error closing the listener: error %v", err) + } + }() + } else { + t.Skip("socket is busy") + } + + cmd := exec.Command("php", "../../tests/client.php", "echo", "unix") + + w, err := NewSocketServer(ls, log).SpawnWorkerWithContext(ctx, cmd) + if err != nil { + t.Fatal(err) + } + go func() { + assert.NoError(t, w.Wait()) + }() + defer func() { + err = w.Stop() + if err != nil { + t.Errorf("error stopping the Process: error %v", err) + } + }() + + res, err := w.Exec(context.Background(), &payload.Payload{Body: []byte("hello")}) + + assert.NoError(t, err) + assert.NotNil(t, res) + assert.NotNil(t, res.Body) + assert.Empty(t, res.Context) + + assert.Equal(t, "hello", res.String()) +} + +func Benchmark_Tcp_SpawnWorker_Stop(b *testing.B) { + ctx := context.Background() + ls, err := net.Listen("tcp", "127.0.0.1:9007") + if err == nil { + defer func() { + err = ls.Close() + if err != nil { + b.Errorf("error closing the listener: error %v", err) + } + }() + } else { + b.Skip("socket is busy") + } + + f := NewSocketServer(ls, log) + for n := 0; n < b.N; n++ { + cmd := exec.Command("php", "../../tests/client.php", "echo", "tcp") + + w, err := f.SpawnWorkerWithContext(ctx, cmd) + if err != nil { + b.Fatal(err) + } + go func() { + assert.NoError(b, w.Wait()) + }() + + err = w.Stop() + if err != nil { + b.Errorf("error stopping the Process: error %v", err) + } + } +} + +func Benchmark_Tcp_Worker_ExecEcho(b *testing.B) { + ctx := context.Background() + ls, err := net.Listen("tcp", "127.0.0.1:9007") + if err == nil { + defer func() { + err = ls.Close() + if err != nil { + b.Errorf("error closing the listener: error %v", err) + } + }() + } else { + b.Skip("socket is busy") + } + + cmd := exec.Command("php", "../../tests/client.php", "echo", "tcp") + + w, err := NewSocketServer(ls, log).SpawnWorkerWithContext(ctx, cmd) + if err != nil { + b.Fatal(err) + } + defer func() { + err = w.Stop() + if err != nil { + b.Errorf("error stopping the Process: error %v", err) + } + }() + + for n := 0; n < b.N; n++ { + if _, err := w.Exec(context.Background(), &payload.Payload{Body: []byte("hello")}); err != nil { + b.Fail() + } + } +} + +func Benchmark_Unix_SpawnWorker_Stop(b *testing.B) { + ctx := context.Background() + ls, err := net.Listen("unix", "sock.unix") + if err == nil { + defer func() { + err = ls.Close() + if err != nil { + b.Errorf("error closing the listener: error %v", err) + } + }() + } else { + b.Skip("socket is busy") + } + + f := NewSocketServer(ls, log) + for n := 0; n < b.N; n++ { + cmd := exec.Command("php", "../../tests/client.php", "echo", "unix") + + w, err := f.SpawnWorkerWithContext(ctx, cmd) + if err != nil { + b.Fatal(err) + } + err = w.Stop() + if err != nil { + b.Errorf("error stopping the Process: error %v", err) + } + } +} + +func Benchmark_Unix_Worker_ExecEcho(b *testing.B) { + ctx := context.Background() + ls, err := net.Listen("unix", "sock.unix") + if err == nil { + defer func() { + err = ls.Close() + if err != nil { + b.Errorf("error closing the listener: error %v", err) + } + }() + } else { + b.Skip("socket is busy") + } + + cmd := exec.Command("php", "../../tests/client.php", "echo", "unix") + + w, err := NewSocketServer(ls, log).SpawnWorkerWithContext(ctx, cmd) + if err != nil { + b.Fatal(err) + } + defer func() { + err = w.Stop() + if err != nil { + b.Errorf("error stopping the Process: error %v", err) + } + }() + + for n := 0; n < b.N; n++ { + if _, err := w.Exec(context.Background(), &payload.Payload{Body: []byte("hello")}); err != nil { + b.Fail() + } + } +} diff --git a/payload/payload.go b/payload/payload.go new file mode 100644 index 0000000..4953e21 --- /dev/null +++ b/payload/payload.go @@ -0,0 +1,27 @@ +package payload + +import ( + "unsafe" +) + +// Payload carries binary header and body to stack and +// back to the server. +type Payload struct { + // Context represent payload context, might be omitted. + Context []byte + // body contains binary payload to be processed by WorkerProcess. + Body []byte + // Type of codec used to decode/encode payload. + Codec byte + // Flags + Flags byte +} + +// String returns payload body as string +func (p *Payload) String() string { + if len(p.Body) == 0 { + return "" + } + + return unsafe.String(unsafe.SliceData(p.Body), len(p.Body)) +} diff --git a/pool/allocator.go b/pool/allocator.go new file mode 100644 index 0000000..9eabbed --- /dev/null +++ b/pool/allocator.go @@ -0,0 +1,82 @@ +package pool + +import ( + "context" + "os/exec" + "time" + + "github.com/roadrunner-server/errors" + "github.com/roadrunner-server/events" + "github.com/roadrunner-server/pool/worker" + "go.uber.org/zap" + "golang.org/x/sync/errgroup" +) + +// Factory is responsible for wrapping given command into tasks WorkerProcess. +type Factory interface { + // SpawnWorkerWithContext creates a new WorkerProcess process based on given command with context. + // Process must not be started. + SpawnWorkerWithContext(context.Context, *exec.Cmd, ...worker.Options) (*worker.Process, error) + // Close the factory and underlying connections. + Close() error +} + +// NewPoolAllocator initializes allocator of the workers +func NewPoolAllocator(ctx context.Context, timeout time.Duration, maxExecs uint64, factory Factory, cmd Command, command []string, log *zap.Logger) func() (*worker.Process, error) { + return func() (*worker.Process, error) { + ctxT, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + w, err := factory.SpawnWorkerWithContext(ctxT, cmd(command), worker.WithLog(log), worker.WithMaxExecs(maxExecs)) + if err != nil { + // context deadline + if errors.Is(errors.TimeOut, err) { + return nil, errors.Str("failed to spawn a worker, possible reasons: https://docs.roadrunner.dev/error-codes/allocate-timeout") + } + return nil, err + } + + // wrap sync worker + log.Debug("worker is allocated", zap.Int64("pid", w.Pid()), zap.Uint64("max_execs", w.MaxExecs()), zap.String("internal_event_name", events.EventWorkerConstruct.String())) + return w, nil + } +} + +// AllocateParallel allocate required number of stack +func AllocateParallel(numWorkers uint64, allocator func() (*worker.Process, error)) ([]*worker.Process, error) { + const op = errors.Op("static_pool_allocate_workers") + + workers := make([]*worker.Process, numWorkers) + eg := new(errgroup.Group) + + // constant number of stack simplify logic + for i := uint64(0); i < numWorkers; i++ { + ii := i + eg.Go(func() error { + w, err := allocator() + if err != nil { + return errors.E(op, errors.WorkerAllocate, err) + } + + workers[ii] = w + return nil + }) + } + + err := eg.Wait() + if err != nil { + for j := 0; j < len(workers); j++ { + jj := j + if workers[jj] != nil { + go func() { + _ = workers[jj].Wait() + }() + + _ = workers[jj].Kill() + } + } + return nil, err + } + + return workers, nil +} diff --git a/pool/command.go b/pool/command.go new file mode 100644 index 0000000..0582978 --- /dev/null +++ b/pool/command.go @@ -0,0 +1,8 @@ +package pool + +import ( + "os/exec" +) + +// Command is a function that returns a new exec.Cmd instance for the given command string. +type Command func(cmd []string) *exec.Cmd diff --git a/pool/config.go b/pool/config.go new file mode 100644 index 0000000..8cd2cea --- /dev/null +++ b/pool/config.go @@ -0,0 +1,83 @@ +package pool + +import ( + "runtime" + "time" +) + +// Config .. Pool config Configures the pool behavior. +type Config struct { + // Debug flag creates new fresh worker before every request. + Debug bool + // Command used to override the server command with the custom one + Command []string `mapstructure:"command"` + // MaxQueueSize is maximum allowed queue size with the pending requests to the workers poll + MaxQueueSize uint64 `mapstructure:"max_queue_size"` + // NumWorkers defines how many sub-processes can be run at once. This value + // might be doubled by Swapper while hot-swap. Defaults to number of CPU cores. + NumWorkers uint64 `mapstructure:"num_workers"` + // MaxJobs defines how many executions is allowed for the worker until + // its destruction. set 1 to create new process for each new task, 0 to let + // worker handle as many tasks as it can. + MaxJobs uint64 `mapstructure:"max_jobs"` + // AllocateTimeout defines for how long pool will be waiting for a worker to + // be freed to handle the task. Defaults to 60s. + AllocateTimeout time.Duration `mapstructure:"allocate_timeout"` + // DestroyTimeout defines for how long pool should be waiting for worker to + // properly destroy, if timeout reached worker will be killed. Defaults to 60s. + DestroyTimeout time.Duration `mapstructure:"destroy_timeout"` + // ResetTimeout defines how long pool should wait before start killing workers + ResetTimeout time.Duration `mapstructure:"reset_timeout"` + // Stream read operation timeout + StreamTimeout time.Duration `mapstructure:"stream_timeout"` + // Supervision config to limit worker and pool memory usage. + Supervisor *SupervisorConfig `mapstructure:"supervisor"` +} + +// InitDefaults enables default config values. +func (cfg *Config) InitDefaults() { + if cfg.NumWorkers == 0 { + cfg.NumWorkers = uint64(runtime.NumCPU()) + } + + if cfg.AllocateTimeout == 0 { + cfg.AllocateTimeout = time.Minute + } + + if cfg.StreamTimeout == 0 { + cfg.StreamTimeout = time.Minute + } + + if cfg.DestroyTimeout == 0 { + cfg.DestroyTimeout = time.Minute + } + + if cfg.ResetTimeout == 0 { + cfg.ResetTimeout = time.Minute + } + + if cfg.Supervisor == nil { + return + } + cfg.Supervisor.InitDefaults() +} + +type SupervisorConfig struct { + // WatchTick defines how often to check the state of worker. + WatchTick time.Duration `mapstructure:"watch_tick"` + // TTL defines the maximum time for the worker is allowed to live. + TTL time.Duration `mapstructure:"ttl"` + // IdleTTL defines the maximum duration worker can spend in idle mode. Disabled when 0. + IdleTTL time.Duration `mapstructure:"idle_ttl"` + // ExecTTL defines maximum lifetime per job. + ExecTTL time.Duration `mapstructure:"exec_ttl"` + // MaxWorkerMemory limits memory per worker. + MaxWorkerMemory uint64 `mapstructure:"max_worker_memory"` +} + +// InitDefaults enables default config values. +func (cfg *SupervisorConfig) InitDefaults() { + if cfg.WatchTick == 0 { + cfg.WatchTick = time.Second * 5 + } +} diff --git a/pool/static_pool/debug.go b/pool/static_pool/debug.go new file mode 100644 index 0000000..a3813e4 --- /dev/null +++ b/pool/static_pool/debug.go @@ -0,0 +1,147 @@ +package static_pool //nolint:stylecheck + +import ( + "context" + "runtime" + + "github.com/roadrunner-server/events" + "github.com/roadrunner-server/goridge/v3/pkg/frame" + "github.com/roadrunner-server/pool/fsm" + "github.com/roadrunner-server/pool/payload" + "go.uber.org/zap" +) + +// execDebug used when debug mode was not set and exec_ttl is 0 +// TODO DRY +func (sp *Pool) execDebug(ctx context.Context, p *payload.Payload, stopCh chan struct{}) (chan *PExec, error) { + sp.log.Debug("executing in debug mode, worker will be destroyed after response is received") + w, err := sp.allocator() + if err != nil { + return nil, err + } + + go func() { + // read the exit status to prevent process to become a zombie + _ = w.Wait() + }() + + rsp, err := w.Exec(ctx, p) + if err != nil { + return nil, err + } + + switch { + case rsp.Flags&frame.STREAM != 0: + // create a channel for the stream (only if there are no errors) + resp := make(chan *PExec, 5) + // send the initial frame + resp <- newPExec(rsp, nil) + + // in case of stream, we should not return worker immediately + go func() { + // would be called on Goexit + defer func() { + sp.log.Debug("stopping [stream] worker", zap.Int("pid", int(w.Pid())), zap.String("state", w.State().String())) + close(resp) + // destroy the worker + errD := w.Stop() + if errD != nil { + sp.log.Debug( + "debug mode: worker stopped with error", + zap.String("reason", "worker error"), + zap.Int64("pid", w.Pid()), + zap.String("internal_event_name", events.EventWorkerError.String()), + zap.Error(errD), + ) + } + }() + + // stream iterator + for { + select { + // we received stop signal + case <-stopCh: + sp.log.Debug("stream stop signal received", zap.Int("pid", int(w.Pid())), zap.String("state", w.State().String())) + ctxT, cancelT := context.WithTimeout(ctx, sp.cfg.StreamTimeout) + err = w.StreamCancel(ctxT) + cancelT() + if err != nil { + w.State().Transition(fsm.StateErrored) + sp.log.Warn("stream cancel error", zap.Error(err)) + } else { + // successfully canceled + w.State().Transition(fsm.StateReady) + sp.log.Debug("transition to the ready state", zap.String("from", w.State().String())) + } + + runtime.Goexit() + default: + // we have to set a stream timeout on every request + switch sp.supervisedExec { + case true: + ctxT, cancelT := context.WithTimeout(context.Background(), sp.cfg.Supervisor.ExecTTL) + pld, next, errI := w.StreamIterWithContext(ctxT) + cancelT() + if errI != nil { + sp.log.Warn("stream error", zap.Error(err)) + + resp <- newPExec(nil, errI) + + // move worker to the invalid state to restart + w.State().Transition(fsm.StateInvalid) + runtime.Goexit() + } + + resp <- newPExec(pld, nil) + + if !next { + w.State().Transition(fsm.StateReady) + // we've got the last frame + runtime.Goexit() + } + case false: + // non supervised execution, can potentially hang here + pld, next, errI := w.StreamIter() + if errI != nil { + sp.log.Warn("stream iter error", zap.Error(err)) + // send error response + resp <- newPExec(nil, errI) + + // move worker to the invalid state to restart + w.State().Transition(fsm.StateInvalid) + runtime.Goexit() + } + + resp <- newPExec(pld, nil) + + if !next { + w.State().Transition(fsm.StateReady) + // we've got the last frame + runtime.Goexit() + } + } + } + } + }() + + return resp, nil + default: + resp := make(chan *PExec, 1) + resp <- newPExec(rsp, nil) + // close the channel + close(resp) + + errD := w.Stop() + if errD != nil { + sp.log.Debug( + "debug mode: worker stopped with error", + zap.String("reason", "worker error"), + zap.Int64("pid", w.Pid()), + zap.String("internal_event_name", events.EventWorkerError.String()), + zap.Error(errD), + ) + } + + return resp, nil + } +} diff --git a/pool/static_pool/fuzz_test.go b/pool/static_pool/fuzz_test.go new file mode 100644 index 0000000..42fcf0c --- /dev/null +++ b/pool/static_pool/fuzz_test.go @@ -0,0 +1,45 @@ +package static_pool //nolint:stylecheck + +import ( + "context" + "os/exec" + "testing" + + "github.com/roadrunner-server/pool/ipc/pipe" + "github.com/roadrunner-server/pool/payload" + "github.com/stretchr/testify/assert" +) + +func FuzzStaticPoolEcho(f *testing.F) { + f.Add([]byte("hello")) + + ctx := context.Background() + p, err := NewPool( + ctx, + func(cmd []string) *exec.Cmd { return exec.Command("php", "../../tests/client.php", "echo", "pipes") }, + pipe.NewPipeFactory(log()), + testCfg, + log(), + ) + assert.NoError(f, err) + assert.NotNil(f, p) + + sc := make(chan struct{}) + f.Fuzz(func(t *testing.T, data []byte) { + // data can't be empty + if len(data) == 0 { + data = []byte("1") + } + + respCh, err := p.Exec(ctx, &payload.Payload{Body: data}, sc) + assert.NoError(t, err) + res := <-respCh + assert.NotNil(t, res) + assert.NotNil(t, res.Body()) + assert.Empty(t, res.Context()) + + assert.Equal(t, data, res.Body()) + }) + + p.Destroy(ctx) +} diff --git a/pool/static_pool/options.go b/pool/static_pool/options.go new file mode 100644 index 0000000..6ce2676 --- /dev/null +++ b/pool/static_pool/options.go @@ -0,0 +1,19 @@ +package static_pool //nolint:stylecheck + +import ( + "go.uber.org/zap" +) + +type Options func(p *Pool) + +func WithLogger(z *zap.Logger) Options { + return func(p *Pool) { + p.log = z + } +} + +func WithQueueSize(l uint64) Options { + return func(p *Pool) { + p.maxQueueSize = l + } +} diff --git a/pool/static_pool/pool.go b/pool/static_pool/pool.go new file mode 100644 index 0000000..8166272 --- /dev/null +++ b/pool/static_pool/pool.go @@ -0,0 +1,418 @@ +package static_pool //nolint:stylecheck + +import ( + "context" + "runtime" + "sync" + "sync/atomic" + "unsafe" + + "github.com/roadrunner-server/errors" + "github.com/roadrunner-server/events" + "github.com/roadrunner-server/goridge/v3/pkg/frame" + "github.com/roadrunner-server/pool/fsm" + "github.com/roadrunner-server/pool/payload" + "github.com/roadrunner-server/pool/pool" + "github.com/roadrunner-server/pool/worker" + workerWatcher "github.com/roadrunner-server/pool/worker_watcher" + "go.uber.org/zap" +) + +const ( + // StopRequest can be sent by worker to indicate that restart is required. + StopRequest = `{"stop":true}` +) + +// Pool controls worker creation, destruction and task routing. Pool uses fixed amount of stack. +type Pool struct { + // pool configuration + cfg *pool.Config + // logger + log *zap.Logger + // worker command creator + cmd pool.Command + // creates and connects to stack + factory pool.Factory + // manages worker states and TTLs + ww *workerWatcher.WorkerWatcher + // allocate new worker + allocator func() (*worker.Process, error) + // exec queue size + queue uint64 + maxQueueSize uint64 + // used in the supervised mode + supervisedExec bool + stopCh chan struct{} + mu sync.RWMutex +} + +// NewPool creates a new worker pool and task multiplexer. Pool will initiate with one worker. If supervisor configuration is provided -> pool will be turned into a supervisedExec mode +func NewPool(ctx context.Context, cmd pool.Command, factory pool.Factory, cfg *pool.Config, log *zap.Logger, options ...Options) (*Pool, error) { + if factory == nil { + return nil, errors.Str("no factory initialized") + } + + if cfg == nil { + return nil, errors.Str("nil configuration provided") + } + + cfg.InitDefaults() + + // for debug mode we need to set the number of workers to 0 (no pre-allocated workers) and max jobs to 1 + if cfg.Debug { + cfg.NumWorkers = 0 + cfg.MaxJobs = 1 + cfg.MaxQueueSize = 0 + } + + p := &Pool{ + cfg: cfg, + cmd: cmd, + factory: factory, + log: log, + queue: 0, + } + + // apply options + for i := 0; i < len(options); i++ { + options[i](p) + } + + if p.log == nil { + var err error + p.log, err = zap.NewDevelopment() + if err != nil { + return nil, err + } + } + + // set up workers allocator + p.allocator = pool.NewPoolAllocator(ctx, p.cfg.AllocateTimeout, p.cfg.MaxJobs, factory, cmd, p.cfg.Command, p.log) + // set up workers watcher + p.ww = workerWatcher.NewSyncWorkerWatcher(p.allocator, p.log, p.cfg.NumWorkers, p.cfg.AllocateTimeout) + + // allocate requested number of workers + workers, err := pool.AllocateParallel(p.cfg.NumWorkers, p.allocator) + if err != nil { + return nil, err + } + + // add workers to the watcher + err = p.ww.Watch(workers) + if err != nil { + return nil, err + } + + if p.cfg.Supervisor != nil { + if p.cfg.Supervisor.ExecTTL != 0 { + // we use supervisedExec ExecWithTTL mode only when ExecTTL is set + // otherwise we may use a faster Exec + p.supervisedExec = true + } + // start the supervisor + p.Start() + } + + return p, nil +} + +// GetConfig returns associated pool configuration. Immutable. +func (sp *Pool) GetConfig() *pool.Config { + return sp.cfg +} + +// Workers returns worker list associated with the pool. +func (sp *Pool) Workers() (workers []*worker.Process) { + return sp.ww.List() +} + +func (sp *Pool) RemoveWorker(ctx context.Context) error { + if sp.cfg.Debug { + sp.log.Warn("remove worker operation is not allowed in debug mode") + return nil + } + var cancel context.CancelFunc + _, ok := ctx.Deadline() + if !ok { + ctx, cancel = context.WithTimeout(ctx, sp.cfg.DestroyTimeout) + defer cancel() + } + + return sp.ww.RemoveWorker(ctx) +} + +func (sp *Pool) AddWorker() error { + if sp.cfg.Debug { + sp.log.Warn("add worker operation is not allowed in debug mode") + return nil + } + return sp.ww.AddWorker() +} + +// Exec executes provided payload on the worker +func (sp *Pool) Exec(ctx context.Context, p *payload.Payload, stopCh chan struct{}) (chan *PExec, error) { + const op = errors.Op("static_pool_exec") + + if len(p.Body) == 0 && len(p.Context) == 0 { + return nil, errors.E(op, errors.Str("payload can not be empty")) + } + + // check if we have space to put the request + if atomic.LoadUint64(&sp.maxQueueSize) != 0 && atomic.LoadUint64(&sp.queue) >= atomic.LoadUint64(&sp.maxQueueSize) { + return nil, errors.E(op, errors.QueueSize, errors.Str("max queue size reached")) + } + + if sp.cfg.Debug { + switch sp.supervisedExec { + case true: + ctxTTL, cancel := context.WithTimeout(ctx, sp.cfg.Supervisor.ExecTTL) + defer cancel() + return sp.execDebug(ctxTTL, p, stopCh) + case false: + return sp.execDebug(context.Background(), p, stopCh) + } + } + + /* + register request in the QUEUE + */ + atomic.AddUint64(&sp.queue, 1) + defer atomic.AddUint64(&sp.queue, ^uint64(0)) + + // see notes at the end of the file +begin: + ctxGetFree, cancel := context.WithTimeout(ctx, sp.cfg.AllocateTimeout) + defer cancel() + w, err := sp.takeWorker(ctxGetFree, op) + if err != nil { + return nil, errors.E(op, err) + } + + var rsp *payload.Payload + switch sp.supervisedExec { + case true: + // in the supervisedExec mode we're limiting the allowed time for the execution inside the PHP worker + ctxT, cancelT := context.WithTimeout(ctx, sp.cfg.Supervisor.ExecTTL) + defer cancelT() + rsp, err = w.Exec(ctxT, p) + case false: + // no context here + // potential problem: if the worker is hung, we can't stop it + rsp, err = w.Exec(context.Background(), p) + } + + if w.MaxExecsReached() { + sp.log.Debug("requests execution limit reached, worker will be restarted", zap.Int64("pid", w.Pid()), zap.Uint64("execs", w.State().NumExecs())) + w.State().Transition(fsm.StateMaxJobsReached) + } + + if err != nil { + // just push event if on any stage was timeout error + switch { + case errors.Is(errors.ExecTTL, err): + // for this case, worker already killed in the ExecTTL function + sp.log.Warn("worker stopped, and will be restarted", zap.String("reason", "execTTL timeout elapsed"), zap.Int64("pid", w.Pid()), zap.String("internal_event_name", events.EventExecTTL.String()), zap.Error(err)) + w.State().Transition(fsm.StateExecTTLReached) + + // worker should already be reallocated + return nil, err + case errors.Is(errors.SoftJob, err): + /* + in case of soft job error, we should not kill the worker, this is just an error payload from the worker. + */ + w.State().Transition(fsm.StateReady) + sp.log.Warn("soft worker error", zap.String("reason", "SoftJob"), zap.Int64("pid", w.Pid()), zap.String("internal_event_name", events.EventWorkerSoftError.String()), zap.Error(err)) + sp.ww.Release(w) + + return nil, err + case errors.Is(errors.Network, err): + // in case of network error, we can't stop the worker, we should kill it + w.State().Transition(fsm.StateErrored) + sp.log.Warn("RoadRunner can't communicate with the worker", zap.String("reason", "worker hung or process was killed"), zap.Int64("pid", w.Pid()), zap.String("internal_event_name", events.EventWorkerError.String()), zap.Error(err)) + // kill the worker instead of sending a net packet to it + _ = w.Kill() + + // do not return it, should be reallocated on Kill + return nil, err + case errors.Is(errors.Retry, err): + // put the worker back to the stack and retry the request with the new one + sp.ww.Release(w) + goto begin + + default: + w.State().Transition(fsm.StateErrored) + sp.log.Warn("worker will be restarted", zap.Int64("pid", w.Pid()), zap.String("internal_event_name", events.EventWorkerDestruct.String()), zap.Error(err)) + + sp.ww.Release(w) + return nil, err + } + } + + // worker want's to be terminated + // unsafe is used to quickly transform []byte to string + if len(rsp.Body) == 0 && unsafe.String(unsafe.SliceData(rsp.Context), len(rsp.Context)) == StopRequest { + w.State().Transition(fsm.StateInvalid) + sp.ww.Release(w) + goto begin + } + + switch { + case rsp.Flags&frame.STREAM != 0: + sp.log.Debug("stream mode", zap.Int64("pid", w.Pid())) + // create channel for the stream (only if there are no errors) + // we need to create a buffered channel to prevent blocking + // stream buffer size should be bigger than regular, to have some payloads ready (optimization) + resp := make(chan *PExec, 5) + // send the initial frame + resp <- newPExec(rsp, nil) + + // in case of stream we should not return worker back immediately + go func() { + // would be called on Goexit + defer func() { + sp.log.Debug("release [stream] worker", zap.Int("pid", int(w.Pid())), zap.String("state", w.State().String())) + close(resp) + sp.ww.Release(w) + }() + + // stream iterator + for { + select { + // we received stop signal + case <-stopCh: + sp.log.Debug("stream stop signal received", zap.Int("pid", int(w.Pid())), zap.String("state", w.State().String())) + ctxT, cancelT := context.WithTimeout(ctx, sp.cfg.StreamTimeout) + err = w.StreamCancel(ctxT) + cancelT() + if err != nil { + w.State().Transition(fsm.StateErrored) + sp.log.Warn("stream cancel error", zap.Error(err)) + } else { + // successfully canceled + w.State().Transition(fsm.StateReady) + sp.log.Debug("transition to the ready state", zap.String("from", w.State().String())) + } + + runtime.Goexit() + default: + // we have to set a stream timeout on every request + switch sp.supervisedExec { + case true: + ctxT, cancelT := context.WithTimeout(context.Background(), sp.cfg.Supervisor.ExecTTL) + pld, next, errI := w.StreamIterWithContext(ctxT) + cancelT() + if errI != nil { + sp.log.Warn("stream error", zap.Error(err)) + + resp <- newPExec(nil, errI) + + // move worker to the invalid state to restart + w.State().Transition(fsm.StateInvalid) + runtime.Goexit() + } + + resp <- newPExec(pld, nil) + + if !next { + w.State().Transition(fsm.StateReady) + // we've got the last frame + runtime.Goexit() + } + case false: + // non supervised execution, can potentially hang here + pld, next, errI := w.StreamIter() + if errI != nil { + sp.log.Warn("stream iter error", zap.Error(err)) + // send error response + resp <- newPExec(nil, errI) + + // move worker to the invalid state to restart + w.State().Transition(fsm.StateInvalid) + runtime.Goexit() + } + + resp <- newPExec(pld, nil) + + if !next { + w.State().Transition(fsm.StateReady) + // we've got the last frame + runtime.Goexit() + } + } + } + } + }() + + return resp, nil + default: + resp := make(chan *PExec, 1) + // send the initial frame + resp <- newPExec(rsp, nil) + sp.log.Debug("req-resp mode", zap.Int64("pid", w.Pid())) + if w.State().Compare(fsm.StateWorking) { + w.State().Transition(fsm.StateReady) + } + // return worker back + sp.ww.Release(w) + // close the channel + close(resp) + return resp, nil + } +} + +func (sp *Pool) QueueSize() uint64 { + return atomic.LoadUint64(&sp.queue) +} + +// Destroy all underlying stack (but let them complete the task). +func (sp *Pool) Destroy(ctx context.Context) { + sp.log.Info("destroy signal received", zap.Duration("timeout", sp.cfg.DestroyTimeout)) + var cancel context.CancelFunc + _, ok := ctx.Deadline() + if !ok { + ctx, cancel = context.WithTimeout(ctx, sp.cfg.DestroyTimeout) + defer cancel() + } + sp.ww.Destroy(ctx) + atomic.StoreUint64(&sp.queue, 0) +} + +func (sp *Pool) Reset(ctx context.Context) error { + // set timeout + ctx, cancel := context.WithTimeout(ctx, sp.cfg.ResetTimeout) + defer cancel() + // reset all workers + numToAllocate := sp.ww.Reset(ctx) + // re-allocate all workers + workers, err := pool.AllocateParallel(numToAllocate, sp.allocator) + if err != nil { + return err + } + // add the NEW workers to the watcher + err = sp.ww.Watch(workers) + if err != nil { + return err + } + + return nil +} + +func (sp *Pool) takeWorker(ctxGetFree context.Context, op errors.Op) (*worker.Process, error) { + // Get function consumes context with timeout + w, err := sp.ww.Take(ctxGetFree) + if err != nil { + // if the error is of kind NoFreeWorkers, it means, that we can't get worker from the stack during the allocate timeout + if errors.Is(errors.NoFreeWorkers, err) { + sp.log.Error( + "no free workers in the pool, wait timeout exceed", + zap.String("reason", "no free workers"), + zap.String("internal_event_name", events.EventNoFreeWorkers.String()), + zap.Error(err), + ) + return nil, errors.E(op, err) + } + // else if err not nil - return error + return nil, errors.E(op, err) + } + return w, nil +} diff --git a/pool/static_pool/pool_test.go b/pool/static_pool/pool_test.go new file mode 100644 index 0000000..0587625 --- /dev/null +++ b/pool/static_pool/pool_test.go @@ -0,0 +1,1173 @@ +package static_pool //nolint:stylecheck + +import ( + "context" + l "log" + "os" + "os/exec" + "runtime" + "strconv" + "sync" + "testing" + "time" + "unsafe" + + "github.com/roadrunner-server/errors" + "github.com/roadrunner-server/pool/fsm" + "github.com/roadrunner-server/pool/ipc/pipe" + "github.com/roadrunner-server/pool/payload" + "github.com/roadrunner-server/pool/pool" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" +) + +var testCfg = &pool.Config{ + NumWorkers: uint64(runtime.NumCPU()), + AllocateTimeout: time.Second * 500, + DestroyTimeout: time.Second * 500, +} + +var log = func() *zap.Logger { + logger, _ := zap.NewDevelopment() + return logger +} + +func Test_NewPool(t *testing.T) { + ctx := context.Background() + p, err := NewPool( + ctx, + func(cmd []string) *exec.Cmd { return exec.Command("php", "../../tests/client.php", "echo", "pipes") }, + pipe.NewPipeFactory(log()), + testCfg, + log(), + ) + assert.NoError(t, err) + assert.NotNil(t, p) + + r, err := p.Exec(ctx, &payload.Payload{Body: []byte("hello"), Context: nil}, make(chan struct{})) + resp := <-r + + assert.Equal(t, []byte("hello"), resp.Body()) + assert.NoError(t, err) + + p.Destroy(ctx) +} + +func Test_NewPoolAddRemoveWorkers(t *testing.T) { + testCfg2 := &pool.Config{ + NumWorkers: 1, + AllocateTimeout: time.Second * 500, + DestroyTimeout: time.Second * 500, + } + + ctx := context.Background() + p, err := NewPool( + ctx, + func(cmd []string) *exec.Cmd { return exec.Command("php", "../../tests/client.php", "echo", "pipes") }, + pipe.NewPipeFactory(log()), + testCfg2, + log(), + ) + assert.NoError(t, err) + assert.NotNil(t, p) + + r, err := p.Exec(ctx, &payload.Payload{Body: []byte("hello"), Context: nil}, make(chan struct{})) + resp := <-r + + assert.Equal(t, []byte("hello"), resp.Body()) + assert.NoError(t, err) + + for i := 0; i < 100; i++ { + err = p.AddWorker() + assert.NoError(t, err) + } + + err = p.AddWorker() + assert.NoError(t, err) + + err = p.RemoveWorker(ctx) + assert.NoError(t, err) + + err = p.RemoveWorker(ctx) + assert.NoError(t, err) + + p.Destroy(ctx) +} + +func Test_StaticPool_NilFactory(t *testing.T) { + ctx := context.Background() + p, err := NewPool( + ctx, + func(cmd []string) *exec.Cmd { return exec.Command("php", "../../tests/client.php", "echo", "pipes") }, + nil, + testCfg, + log(), + ) + assert.Error(t, err) + assert.Nil(t, p) +} + +func Test_StaticPool_NilConfig(t *testing.T) { + ctx := context.Background() + p, err := NewPool( + ctx, + func(cmd []string) *exec.Cmd { return exec.Command("php", "../../tests/client.php", "echo", "pipes") }, + pipe.NewPipeFactory(log()), + nil, + log(), + ) + assert.Error(t, err) + assert.Nil(t, p) +} + +func Test_StaticPool_ImmediateDestroy(t *testing.T) { + ctx := context.Background() + + p, err := NewPool( + ctx, + func(cmd []string) *exec.Cmd { return exec.Command("php", "../../tests/client.php", "echo", "pipes") }, + pipe.NewPipeFactory(log()), + testCfg, + log(), + ) + assert.NoError(t, err) + assert.NotNil(t, p) + + _, _ = p.Exec(ctx, &payload.Payload{Body: []byte("hello"), Context: nil}, make(chan struct{})) + + ctx, cancel := context.WithTimeout(ctx, time.Nanosecond) + defer cancel() + + p.Destroy(ctx) +} + +func Test_StaticPool_RemoveWorker(t *testing.T) { + ctx := context.Background() + + testCfg2 := &pool.Config{ + NumWorkers: 5, + AllocateTimeout: time.Second * 5, + DestroyTimeout: time.Second * 5, + } + + p, err := NewPool( + ctx, + func(cmd []string) *exec.Cmd { return exec.Command("php", "../../tests/client.php", "echo", "pipes") }, + pipe.NewPipeFactory(log()), + testCfg2, + log(), + ) + assert.NoError(t, err) + assert.NotNil(t, p) + + _, err = p.Exec(ctx, &payload.Payload{Body: []byte("hello"), Context: nil}, make(chan struct{})) + assert.NoError(t, err) + + wrks := p.Workers() + for i := 0; i < len(wrks); i++ { + assert.NoError(t, p.RemoveWorker(ctx)) + } + + _, err = p.Exec(ctx, &payload.Payload{Body: []byte("hello"), Context: nil}, make(chan struct{})) + assert.Error(t, err) + + err = p.AddWorker() + assert.NoError(t, err) + + _, err = p.Exec(ctx, &payload.Payload{Body: []byte("hello"), Context: nil}, make(chan struct{})) + assert.NoError(t, err) + + // after removing all workers, we should have 1 worker + 1 we added + assert.Len(t, p.Workers(), 2) + + p.Destroy(ctx) +} + +func Test_Pool_Reallocate(t *testing.T) { + testCfg2 := &pool.Config{ + NumWorkers: 1, + AllocateTimeout: time.Second * 500, + DestroyTimeout: time.Second * 500, + } + + ctx := context.Background() + p, err := NewPool( + ctx, + func(cmd []string) *exec.Cmd { return exec.Command("php", "../../tests/client.php", "echo", "pipes") }, + pipe.NewPipeFactory(log()), + testCfg2, + log(), + ) + require.NoError(t, err) + require.NotNil(t, p) + + wg := sync.WaitGroup{} + wg.Add(1) + + require.NoError(t, os.Rename("../../tests/client.php", "../../tests/client.bak")) + + go func() { + for i := 0; i < 50; i++ { + time.Sleep(time.Millisecond * 100) + _, errResp := p.Exec(ctx, &payload.Payload{Body: []byte("hello"), Context: nil}, make(chan struct{})) + require.NoError(t, errResp) + } + wg.Done() + }() + + _ = p.Workers()[0].Kill() + + time.Sleep(time.Second * 5) + require.NoError(t, os.Rename("../../tests/client.bak", "../../tests/client.php")) + + wg.Wait() + + t.Cleanup(func() { + p.Destroy(ctx) + }) +} + +func Test_NewPoolReset(t *testing.T) { + ctx := context.Background() + + testCfg2 := &pool.Config{ + NumWorkers: 1, + AllocateTimeout: time.Second * 500, + DestroyTimeout: time.Second * 500, + } + + p, err := NewPool( + ctx, + func(cmd []string) *exec.Cmd { return exec.Command("php", "../../tests/client.php", "echo", "pipes") }, + pipe.NewPipeFactory(log()), + testCfg2, + log(), + ) + assert.NoError(t, err) + assert.NotNil(t, p) + + w := p.Workers() + if len(w) == 0 { + t.Fatal("should be workers inside") + } + pid := w[0].Pid() + + pldd, err := p.Exec(ctx, &payload.Payload{Body: []byte("hello"), Context: nil}, make(chan struct{})) + require.NoError(t, err) + pld := <-pldd + require.NotNil(t, pld.Body()) + + wg := &sync.WaitGroup{} + wg.Add(1) + go func() { + for i := 0; i < 100; i++ { + time.Sleep(time.Millisecond * 10) + pldG, errG := p.Exec(ctx, &payload.Payload{Body: []byte("hello"), Context: nil}, make(chan struct{})) + require.NoError(t, errG) + pldGG := <-pldG + require.NotNil(t, pldGG.Body()) + } + + wg.Done() + }() + + require.NoError(t, p.Reset(context.Background())) + + pldd, err = p.Exec(ctx, &payload.Payload{Body: []byte("hello"), Context: nil}, make(chan struct{})) + require.NoError(t, err) + pld = <-pldd + require.NotNil(t, pld.Body()) + + w2 := p.Workers() + if len(w2) == 0 { + t.Fatal("should be workers inside") + } + + require.NotEqual(t, pid, w2[0].Pid()) + wg.Wait() + p.Destroy(ctx) +} + +func Test_StaticPool_Invalid(t *testing.T) { + p, err := NewPool( + context.Background(), + func(cmd []string) *exec.Cmd { return exec.Command("php", "../../tests/invalid.php") }, + pipe.NewPipeFactory(log()), + testCfg, + log(), + ) + + assert.Nil(t, p) + assert.Error(t, err) +} + +func Test_ConfigNoErrorInitDefaults(t *testing.T) { + p, err := NewPool( + context.Background(), + func(cmd []string) *exec.Cmd { return exec.Command("php", "../../tests/client.php", "echo", "pipes") }, + pipe.NewPipeFactory(log()), + &pool.Config{ + AllocateTimeout: time.Second, + DestroyTimeout: time.Second, + }, + log(), + ) + + assert.NotNil(t, p) + assert.NoError(t, err) + p.Destroy(context.Background()) +} + +func Test_StaticPool_QueueSizeLimit(t *testing.T) { + testCfg2 := &pool.Config{ + NumWorkers: 1, + AllocateTimeout: time.Second * 500, + DestroyTimeout: time.Second * 500, + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second*500) + defer cancel() + + p, err := NewPool( + ctx, + // sleep for 10 seconds + func(cmd []string) *exec.Cmd { return exec.Command("php", "../../tests/sleep-ttl.php") }, + pipe.NewPipeFactory(log()), + testCfg2, + log(), + WithQueueSize(1), + ) + require.NoError(t, err) + + defer p.Destroy(ctx) + + assert.NotNil(t, p) + wg := &sync.WaitGroup{} + wg.Add(2) + + go func() { + time.Sleep(time.Second * 2) + _, err1 := p.Exec(ctx, &payload.Payload{Body: []byte("hello")}, make(chan struct{})) + require.Error(t, err1) + wg.Done() + }() + go func() { + time.Sleep(time.Second * 2) + _, err2 := p.Exec(ctx, &payload.Payload{Body: []byte("hello")}, make(chan struct{})) + require.Error(t, err2) + wg.Done() + }() + + re, err := p.Exec(ctx, &payload.Payload{Body: []byte("hello")}, make(chan struct{})) + res := <-re + + assert.NoError(t, err) + assert.NotNil(t, res) + assert.NotNil(t, res.Body()) + assert.Empty(t, res.Context()) + + assert.Equal(t, "hello world", res.Payload().String()) + wg.Wait() + + p.Destroy(ctx) +} + +func Test_StaticPool_Echo(t *testing.T) { + ctx := context.Background() + p, err := NewPool( + ctx, + func(cmd []string) *exec.Cmd { return exec.Command("php", "../../tests/client.php", "echo", "pipes") }, + pipe.NewPipeFactory(log()), + testCfg, + log(), + ) + assert.NoError(t, err) + + defer p.Destroy(ctx) + + assert.NotNil(t, p) + + re, err := p.Exec(ctx, &payload.Payload{Body: []byte("hello")}, make(chan struct{})) + + res := <-re + + assert.NoError(t, err) + assert.NotNil(t, res) + assert.NotNil(t, res.Body()) + assert.Empty(t, res.Context()) + + assert.Equal(t, "hello", res.Payload().String()) +} + +func Test_StaticPool_Echo_NilContext(t *testing.T) { + ctx := context.Background() + p, err := NewPool( + ctx, + func(cmd []string) *exec.Cmd { return exec.Command("php", "../../tests/client.php", "echo", "pipes") }, + pipe.NewPipeFactory(log()), + testCfg, + log(), + ) + assert.NoError(t, err) + + defer p.Destroy(ctx) + + assert.NotNil(t, p) + + re, err := p.Exec(ctx, &payload.Payload{Body: []byte("hello"), Context: nil}, make(chan struct{})) + assert.NoError(t, err) + + res := <-re + + assert.NotNil(t, res) + assert.NotNil(t, res.Body()) + assert.Empty(t, res.Context()) + + assert.Equal(t, "hello", res.Payload().String()) +} + +func Test_StaticPool_Echo_Context(t *testing.T) { + ctx := context.Background() + p, err := NewPool( + ctx, + func(cmd []string) *exec.Cmd { return exec.Command("php", "../../tests/client.php", "head", "pipes") }, + pipe.NewPipeFactory(log()), + testCfg, + log(), + ) + assert.NoError(t, err) + + defer p.Destroy(ctx) + + assert.NotNil(t, p) + + re, err := p.Exec(ctx, &payload.Payload{Body: []byte("hello"), Context: []byte("world")}, make(chan struct{})) + assert.NoError(t, err) + + res := <-re + + assert.NotNil(t, res) + assert.Empty(t, res.Body()) + assert.NotNil(t, res.Context()) + + assert.Equal(t, "world", string(res.Context())) +} + +func Test_StaticPool_JobError(t *testing.T) { + ctx := context.Background() + p, err := NewPool( + ctx, + func(cmd []string) *exec.Cmd { return exec.Command("php", "../../tests/client.php", "error", "pipes") }, + pipe.NewPipeFactory(log()), + testCfg, + log(), + ) + assert.NoError(t, err) + assert.NotNil(t, p) + + time.Sleep(time.Second * 2) + + res, err := p.Exec(ctx, &payload.Payload{Body: []byte("hello")}, make(chan struct{})) + assert.Error(t, err) + assert.Nil(t, res) + + if errors.Is(errors.SoftJob, err) == false { + t.Fatal("error should be of type errors.Exec") + } + + assert.Contains(t, err.Error(), "hello") + p.Destroy(ctx) +} + +func Test_StaticPool_Broken_Replace(t *testing.T) { + ctx := context.Background() + + z, err := zap.NewProduction() + require.NoError(t, err) + + p, err := NewPool( + ctx, + func(cmd []string) *exec.Cmd { return exec.Command("php", "../../tests/client.php", "broken", "pipes") }, + pipe.NewPipeFactory(log()), + testCfg, + z, + ) + assert.NoError(t, err) + assert.NotNil(t, p) + + time.Sleep(time.Second) + res, err := p.Exec(ctx, &payload.Payload{Body: []byte("hello")}, make(chan struct{})) + assert.Error(t, err) + assert.Nil(t, res) + + p.Destroy(ctx) +} + +func Test_StaticPool_Broken_FromOutside(t *testing.T) { + ctx := context.Background() + + cfg2 := &pool.Config{ + NumWorkers: 1, + AllocateTimeout: time.Second * 5, + DestroyTimeout: time.Second * 5, + } + + p, err := NewPool( + ctx, + func(cmd []string) *exec.Cmd { return exec.Command("php", "../../tests/client.php", "echo", "pipes") }, + pipe.NewPipeFactory(log()), + cfg2, + nil, + ) + assert.NoError(t, err) + assert.NotNil(t, p) + time.Sleep(time.Second) + + re, err := p.Exec(ctx, &payload.Payload{Body: []byte("hello")}, make(chan struct{})) + assert.NoError(t, err) + + res := <-re + + assert.NotNil(t, res) + assert.NotNil(t, res.Body()) + assert.Empty(t, res.Context()) + + assert.Equal(t, "hello", res.Payload().String()) + assert.Equal(t, 1, len(p.Workers())) + + // first creation + time.Sleep(time.Second * 2) + // killing random worker and expecting pool to replace it + err = p.Workers()[0].Kill() + if err != nil { + t.Errorf("error killing the process: error %v", err) + } + + // re-creation + time.Sleep(time.Second * 2) + list := p.Workers() + for _, w := range list { + assert.Equal(t, fsm.StateReady, w.State().CurrentState()) + } + p.Destroy(context.Background()) +} + +func Test_StaticPool_AllocateTimeout(t *testing.T) { + p, err := NewPool( + context.Background(), + func(cmd []string) *exec.Cmd { return exec.Command("php", "../../tests/client.php", "delay", "pipes") }, + pipe.NewPipeFactory(log()), + &pool.Config{ + NumWorkers: 1, + AllocateTimeout: time.Nanosecond * 1, + DestroyTimeout: time.Second * 2, + }, + log(), + ) + assert.Error(t, err) + if !errors.Is(errors.WorkerAllocate, err) { + t.Fatal("error should be of type WorkerAllocate") + } + assert.Nil(t, p) +} + +func Test_StaticPool_Replace_Worker(t *testing.T) { + ctx := context.Background() + p, err := NewPool( + ctx, + func(cmd []string) *exec.Cmd { return exec.Command("php", "../../tests/client.php", "pid", "pipes") }, + pipe.NewPipeFactory(log()), + &pool.Config{ + NumWorkers: 1, + MaxJobs: 1, + AllocateTimeout: time.Second * 5, + DestroyTimeout: time.Second, + }, + log(), + ) + assert.NoError(t, err) + assert.NotNil(t, p) + + // prevent process is not ready + time.Sleep(time.Second) + + var lastPID string + lastPID = strconv.Itoa(int(p.Workers()[0].Pid())) + + re, err := p.Exec(ctx, &payload.Payload{Body: []byte("hello")}, make(chan struct{})) + res := <-re + require.Equal(t, lastPID, string(res.Body())) + require.NoError(t, err) + + for i := 0; i < 10; i++ { + re, err = p.Exec(ctx, &payload.Payload{Body: []byte("hello")}, make(chan struct{})) + require.NoError(t, err) + + res = <-re + + require.NotNil(t, res) + require.NotNil(t, res.Body()) + require.Empty(t, res.Context()) + + require.NotEqual(t, lastPID, string(res.Body())) + lastPID = string(res.Body()) + } + + p.Destroy(context.Background()) +} + +func Test_StaticPool_DebugAddRemove(t *testing.T) { + ctx := context.Background() + p, err := NewPool( + ctx, + func(cmd []string) *exec.Cmd { return exec.Command("php", "../../tests/client.php", "pid", "pipes") }, + pipe.NewPipeFactory(log()), + &pool.Config{ + Debug: true, + AllocateTimeout: time.Second, + DestroyTimeout: time.Second, + }, + log(), + ) + assert.NoError(t, err) + assert.NotNil(t, p) + + // prevent process is not ready + time.Sleep(time.Second) + assert.Len(t, p.Workers(), 0) + + var lastPID string + re, err := p.Exec(ctx, &payload.Payload{Body: []byte("hello")}, make(chan struct{})) + assert.NoError(t, err) + + res := <-re + + assert.NotEqual(t, lastPID, string(res.Body())) + + assert.Len(t, p.Workers(), 0) + + err = p.AddWorker() + assert.NoError(t, err) + + assert.Len(t, p.Workers(), 0) + + ctxT, cancel := context.WithTimeout(ctx, time.Microsecond) + err = p.RemoveWorker(ctxT) + cancel() + assert.NoError(t, err) + + p.Destroy(context.Background()) +} + +func Test_StaticPool_Debug_Worker(t *testing.T) { + ctx := context.Background() + p, err := NewPool( + ctx, + func(cmd []string) *exec.Cmd { return exec.Command("php", "../../tests/client.php", "pid", "pipes") }, + pipe.NewPipeFactory(log()), + &pool.Config{ + Debug: true, + AllocateTimeout: time.Second, + DestroyTimeout: time.Second, + }, + log(), + ) + assert.NoError(t, err) + assert.NotNil(t, p) + + // prevent process is not ready + time.Sleep(time.Second) + assert.Len(t, p.Workers(), 0) + + var lastPID string + re, err := p.Exec(ctx, &payload.Payload{Body: []byte("hello")}, make(chan struct{})) + assert.NoError(t, err) + + res := <-re + + assert.NotEqual(t, lastPID, string(res.Body())) + + assert.Len(t, p.Workers(), 0) + + for i := 0; i < 10; i++ { + assert.Len(t, p.Workers(), 0) + re, err = p.Exec(ctx, &payload.Payload{Body: []byte("hello")}, make(chan struct{})) + + res = <-re + + assert.NoError(t, err) + assert.NotNil(t, res) + assert.NotNil(t, res.Body()) + assert.Empty(t, res.Context()) + + assert.NotEqual(t, lastPID, string(res.Body())) + lastPID = string(res.Body()) + } + + p.Destroy(context.Background()) +} + +// identical to replace but controlled on worker side +func Test_Static_Pool_Destroy_And_Close(t *testing.T) { + ctx := context.Background() + p, err := NewPool( + ctx, + func(cmd []string) *exec.Cmd { return exec.Command("php", "../../tests/client.php", "delay", "pipes") }, + pipe.NewPipeFactory(log()), + &pool.Config{ + NumWorkers: 1, + AllocateTimeout: time.Second, + DestroyTimeout: time.Second, + }, + log(), + ) + + assert.NotNil(t, p) + assert.NoError(t, err) + + p.Destroy(ctx) + _, err = p.Exec(ctx, &payload.Payload{Body: []byte("100")}, make(chan struct{})) + assert.Error(t, err) +} + +// identical to replace but controlled on worker side +func Test_Static_Pool_Destroy_And_Close_While_Wait(t *testing.T) { + ctx := context.Background() + p, err := NewPool( + ctx, + func(cmd []string) *exec.Cmd { return exec.Command("php", "../../tests/client.php", "delay", "pipes") }, + pipe.NewPipeFactory(log()), + &pool.Config{ + NumWorkers: 1, + AllocateTimeout: time.Second, + DestroyTimeout: time.Second, + }, + log(), + ) + + assert.NotNil(t, p) + assert.NoError(t, err) + + go func() { + _, errP := p.Exec(ctx, &payload.Payload{Body: []byte("100")}, make(chan struct{})) + if errP != nil { + t.Errorf("error executing payload: error %v", err) + } + }() + time.Sleep(time.Millisecond * 100) + + p.Destroy(ctx) + _, err = p.Exec(ctx, &payload.Payload{Body: []byte("100")}, make(chan struct{})) + assert.Error(t, err) +} + +// identical to replace but controlled on worker side +func Test_Static_Pool_Handle_Dead(t *testing.T) { + ctx := context.Background() + p, err := NewPool( + context.Background(), + func(cmd []string) *exec.Cmd { + return exec.Command("php", "../../tests/slow-destroy.php", "echo", "pipes") + }, + pipe.NewPipeFactory(log()), + &pool.Config{ + NumWorkers: 5, + AllocateTimeout: time.Second * 100, + DestroyTimeout: time.Second, + }, + log(), + ) + assert.NoError(t, err) + assert.NotNil(t, p) + + time.Sleep(time.Second) + for i := range p.Workers() { + p.Workers()[i].State().Transition(fsm.StateErrored) + } + + _, err = p.Exec(ctx, &payload.Payload{Body: []byte("hello")}, make(chan struct{})) + assert.NoError(t, err) + p.Destroy(ctx) +} + +// identical to replace but controlled on worker side +func Test_Static_Pool_Slow_Destroy(t *testing.T) { + p, err := NewPool( + context.Background(), + func(cmd []string) *exec.Cmd { + return exec.Command("php", "../../tests/slow-destroy.php", "echo", "pipes") + }, + pipe.NewPipeFactory(log()), + &pool.Config{ + NumWorkers: 5, + AllocateTimeout: time.Second, + DestroyTimeout: time.Second, + }, + log(), + ) + + assert.NoError(t, err) + assert.NotNil(t, p) + + p.Destroy(context.Background()) +} + +func Test_StaticPool_ResetTimeout(t *testing.T) { + ctx := context.Background() + + p, err := NewPool( + ctx, + // sleep for the 3 seconds + func(cmd []string) *exec.Cmd { return exec.Command("php", "../../tests/sleep.php", "pipes") }, + pipe.NewPipeFactory(log()), + &pool.Config{ + Debug: false, + NumWorkers: 2, + AllocateTimeout: time.Second * 100, + DestroyTimeout: time.Second * 100, + ResetTimeout: time.Second * 3, + }, + log(), + ) + assert.NoError(t, err) + assert.NotNil(t, p) + + go func() { + _, _ = p.Exec(ctx, &payload.Payload{Body: []byte("hello")}, make(chan struct{})) + }() + + time.Sleep(time.Second) + + err = p.Reset(ctx) + assert.NoError(t, err) + + t.Cleanup(func() { + p.Destroy(ctx) + }) +} + +func Test_StaticPool_NoFreeWorkers(t *testing.T) { + ctx := context.Background() + + p, err := NewPool( + ctx, + // sleep for the 3 seconds + func(cmd []string) *exec.Cmd { return exec.Command("php", "../../tests/sleep.php", "pipes") }, + pipe.NewPipeFactory(log()), + &pool.Config{ + Debug: false, + NumWorkers: 1, + AllocateTimeout: time.Second, + DestroyTimeout: time.Second, + Supervisor: nil, + }, + log(), + ) + assert.NoError(t, err) + assert.NotNil(t, p) + + go func() { + _, _ = p.Exec(ctx, &payload.Payload{Body: []byte("hello")}, make(chan struct{})) + }() + + time.Sleep(time.Second) + res, err := p.Exec(ctx, &payload.Payload{Body: []byte("hello")}, make(chan struct{})) + assert.Error(t, err) + assert.Nil(t, res) + + time.Sleep(time.Second) + + p.Destroy(ctx) +} + +// identical to replace but controlled on worker side +func Test_StaticPool_Stop_Worker(t *testing.T) { + ctx := context.Background() + p, err := NewPool( + ctx, + func(cmd []string) *exec.Cmd { return exec.Command("php", "../../tests/client.php", "stop", "pipes") }, + pipe.NewPipeFactory(log()), + &pool.Config{ + NumWorkers: 1, + AllocateTimeout: time.Second, + DestroyTimeout: time.Second, + }, + log(), + ) + assert.NoError(t, err) + assert.NotNil(t, p) + + defer p.Destroy(ctx) + time.Sleep(time.Second) + + var lastPID string + lastPID = strconv.Itoa(int(p.Workers()[0].Pid())) + + re, err := p.Exec(ctx, &payload.Payload{Body: []byte("hello")}, make(chan struct{})) + assert.NoError(t, err) + + res := <-re + + assert.Equal(t, lastPID, string(res.Body())) + + for i := 0; i < 10; i++ { + re, err := p.Exec(ctx, &payload.Payload{Body: []byte("hello")}, make(chan struct{})) + + res := <-re + + assert.NoError(t, err) + assert.NotNil(t, res) + assert.NotNil(t, res.Body()) + assert.Empty(t, res.Context()) + + assert.NotEqual(t, lastPID, string(res.Body())) + lastPID = string(res.Body()) + } +} + +func Test_StaticPool_QueueSize(t *testing.T) { + ctx := context.Background() + + p, err := NewPool( + ctx, + // sleep for the 3 seconds + func(cmd []string) *exec.Cmd { return exec.Command("php", "../../tests/sleep_short.php", "pipes") }, + pipe.NewPipeFactory(log()), + &pool.Config{ + Debug: false, + NumWorkers: 1, + AllocateTimeout: time.Second, + DestroyTimeout: time.Second, + Supervisor: nil, + }, + log(), + ) + assert.NoError(t, err) + assert.NotNil(t, p) + + for i := 0; i < 10; i++ { + go func() { + _, _ = p.Exec(ctx, &payload.Payload{Body: []byte("hello")}, make(chan struct{})) + }() + } + + time.Sleep(time.Second) + require.LessOrEqual(t, p.QueueSize(), uint64(10)) + time.Sleep(time.Second * 20) + require.Less(t, p.QueueSize(), uint64(10)) + + p.Destroy(ctx) +} + +// identical to replace but controlled on worker side +func Test_Static_Pool_WrongCommand1(t *testing.T) { + p, err := NewPool( + context.Background(), + func(cmd []string) *exec.Cmd { + return exec.Command("phg", "../../tests/slow-destroy.php", "echo", "pipes") + }, + pipe.NewPipeFactory(log()), + &pool.Config{ + NumWorkers: 5, + AllocateTimeout: time.Second, + DestroyTimeout: time.Second, + }, + log(), + ) + + assert.Error(t, err) + assert.Nil(t, p) +} + +// identical to replace but controlled on worker side +func Test_Static_Pool_WrongCommand2(t *testing.T) { + p, err := NewPool( + context.Background(), + func(cmd []string) *exec.Cmd { return exec.Command("php", "", "echo", "pipes") }, + pipe.NewPipeFactory(log()), + &pool.Config{ + NumWorkers: 5, + AllocateTimeout: time.Second, + DestroyTimeout: time.Second, + }, + log(), + ) + + assert.Error(t, err) + assert.Nil(t, p) +} + +func Test_CRC_WithPayload(t *testing.T) { + ctx := context.Background() + p, err := NewPool( + ctx, + func(cmd []string) *exec.Cmd { return exec.Command("php", "../../tests/crc_error.php") }, + pipe.NewPipeFactory(log()), + testCfg, + log(), + ) + assert.Error(t, err) + data := err.Error() + assert.Contains(t, data, "warning: some weird php erro") + require.Nil(t, p) +} + +/* + PTR: + +Benchmark_Pool_Echo-32 49076 29926 ns/op 8016 B/op 20 allocs/op +Benchmark_Pool_Echo-32 47257 30779 ns/op 8047 B/op 20 allocs/op +Benchmark_Pool_Echo-32 46737 29440 ns/op 8065 B/op 20 allocs/op +Benchmark_Pool_Echo-32 51177 29074 ns/op 7981 B/op 20 allocs/op +Benchmark_Pool_Echo-32 51764 28319 ns/op 8012 B/op 20 allocs/op +Benchmark_Pool_Echo-32 54054 30714 ns/op 7987 B/op 20 allocs/op +Benchmark_Pool_Echo-32 54391 30689 ns/op 8055 B/op 20 allocs/op + +VAL: +Benchmark_Pool_Echo-32 47936 28679 ns/op 7942 B/op 19 allocs/op +Benchmark_Pool_Echo-32 49010 29830 ns/op 7970 B/op 19 allocs/op +Benchmark_Pool_Echo-32 46771 29031 ns/op 8014 B/op 19 allocs/op +Benchmark_Pool_Echo-32 47760 30517 ns/op 7955 B/op 19 allocs/op +Benchmark_Pool_Echo-32 48148 29816 ns/op 7950 B/op 19 allocs/op +Benchmark_Pool_Echo-32 52705 29809 ns/op 7979 B/op 19 allocs/op +Benchmark_Pool_Echo-32 54374 27776 ns/op 7947 B/op 19 allocs/op +*/ +func Benchmark_Pool_Echo(b *testing.B) { + ctx := context.Background() + p, err := NewPool( + ctx, + func(cmd []string) *exec.Cmd { return exec.Command("php", "../../tests/client.php", "echo", "pipes") }, + pipe.NewPipeFactory(log()), + testCfg, + log(), + ) + if err != nil { + b.Fatal(err) + } + + bd := make([]byte, 1024) + c := make([]byte, 1024) + + pld := &payload.Payload{ + Context: c, + Body: bd, + } + + b.ResetTimer() + b.ReportAllocs() + sc := make(chan struct{}) + for n := 0; n < b.N; n++ { + _, err = p.Exec(ctx, pld, sc) + assert.NoError(b, err) + } +} + +// Benchmark_Pool_Echo_Batched-32 366996 2873 ns/op 1233 B/op 24 allocs/op +// PTR -> Benchmark_Pool_Echo_Batched-32 406839 2900 ns/op 1059 B/op 23 allocs/op +// PTR -> Benchmark_Pool_Echo_Batched-32 413312 2904 ns/op 1067 B/op 23 allocs/op +func Benchmark_Pool_Echo_Batched(b *testing.B) { + ctx := context.Background() + p, err := NewPool( + ctx, + func(cmd []string) *exec.Cmd { return exec.Command("php", "../../tests/client.php", "echo", "pipes") }, + pipe.NewPipeFactory(log()), + &pool.Config{ + NumWorkers: uint64(runtime.NumCPU()), + AllocateTimeout: time.Second * 100, + DestroyTimeout: time.Second, + }, + log(), + ) + assert.NoError(b, err) + defer p.Destroy(ctx) + + bd := make([]byte, 1024) + c := make([]byte, 1024) + + pld := &payload.Payload{ + Context: c, + Body: bd, + } + + b.ResetTimer() + b.ReportAllocs() + + var wg sync.WaitGroup + sc := make(chan struct{}) + for i := 0; i < b.N; i++ { + wg.Add(1) + go func() { + defer wg.Done() + if _, err := p.Exec(ctx, pld, sc); err != nil { + b.Fail() + l.Println(err) + } + }() + } + + wg.Wait() +} + +// Benchmark_Pool_Echo_Replaced-32 104/100 10900218 ns/op 52365 B/op 125 allocs/op +func Benchmark_Pool_Echo_Replaced(b *testing.B) { + ctx := context.Background() + p, err := NewPool( + ctx, + func(cmd []string) *exec.Cmd { return exec.Command("php", "../../tests/client.php", "echo", "pipes") }, + pipe.NewPipeFactory(log()), + &pool.Config{ + NumWorkers: 1, + MaxJobs: 1, + AllocateTimeout: time.Second, + DestroyTimeout: time.Second, + }, + log(), + ) + assert.NoError(b, err) + defer p.Destroy(ctx) + b.ResetTimer() + b.ReportAllocs() + + sc := make(chan struct{}) + for n := 0; n < b.N; n++ { + if _, err := p.Exec(ctx, &payload.Payload{Body: []byte("hello")}, sc); err != nil { + b.Fail() + l.Println(err) + } + } +} + +// BenchmarkToStringUnsafe-12 566317729 1.91 ns/op 0 B/op 0 allocs/op +// BenchmarkToStringUnsafe-32 1000000000 0.4434 ns/op 0 B/op 0 allocs/op +func BenchmarkToStringUnsafe(b *testing.B) { + testPayload := []byte( + "falsflasjlifjwpoihejfoiwejow{}{}{}{}jelfjasjfhwaopiehjtopwhtgohrgouahsgkljasdlfjasl;fjals;jdflkndgouwhetopwqhjtojfalsflasjlifjwpoihejfoiwejow{}{}{}{}jelfjasjfhwaopiehjtopwhtgohrgouahsgkljasdlfjasl;fjals;jdflkndgouwhetopwqhjtojfalsflasjlifjwpoihejfoiwejow{}{}{}{}jelfjasjfhwaopiehjtopwhtgohrgouahsgkljasdlfjasl;fjals;jdflkndgouwhetopwqhjtojfalsflasjlifjwpoihejfoiwejow{}{}{}{}jelfjasjfhwaopiehjtopwhtgohrgouahsgkljasdlfjasl;fjals;jdflkndgouwhetopwqhjtojfalsflasjlifjwpoihejfoiwejow{}{}{}{}jelfjasjfhwaopiehjtopwhtgohrgouahsgkljasdlfjasl;fjals;jdflkndgouwhetopwqhjtojfalsflasjlifjwpoihejfoiwejow{}{}{}{}jelfjasjfhwaopiehjtopwhtgohrgouahsgkljasdlfjasl;fjals;jdflkndgouwhetopwqhjtojfalsflasjlifjwpoihejfoiwejow{}{}{}{}jelfjasjfhwaopiehjtopwhtgohrgouahsgkljasdlfjasl;fjals;jdflkndgouwhetopwqhjtoj", + ) + b.ResetTimer() + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + res := unsafe.String(unsafe.SliceData(testPayload), len(testPayload)) + _ = res + } +} + +// BenchmarkToStringSafe-32 8017846 182.5 ns/op 896 B/op 1 allocs/op +// inline BenchmarkToStringSafe-12 28926276 46.6 ns/op 128 B/op 1 allocs/op +func BenchmarkToStringSafe(b *testing.B) { + testPayload := []byte( + "falsflasjlifjwpoihejfoiwejow{}{}{}{}jelfjasjfhwaopiehjtopwhtgohrgouahsgkljasdlfjasl;fjals;jdflkndgouwhetopwqhjtojfalsflasjlifjwpoihejfoiwejow{}{}{}{}jelfjasjfhwaopiehjtopwhtgohrgouahsgkljasdlfjasl;fjals;jdflkndgouwhetopwqhjtojfalsflasjlifjwpoihejfoiwejow{}{}{}{}jelfjasjfhwaopiehjtopwhtgohrgouahsgkljasdlfjasl;fjals;jdflkndgouwhetopwqhjtojfalsflasjlifjwpoihejfoiwejow{}{}{}{}jelfjasjfhwaopiehjtopwhtgohrgouahsgkljasdlfjasl;fjals;jdflkndgouwhetopwqhjtojfalsflasjlifjwpoihejfoiwejow{}{}{}{}jelfjasjfhwaopiehjtopwhtgohrgouahsgkljasdlfjasl;fjals;jdflkndgouwhetopwqhjtojfalsflasjlifjwpoihejfoiwejow{}{}{}{}jelfjasjfhwaopiehjtopwhtgohrgouahsgkljasdlfjasl;fjals;jdflkndgouwhetopwqhjtojfalsflasjlifjwpoihejfoiwejow{}{}{}{}jelfjasjfhwaopiehjtopwhtgohrgouahsgkljasdlfjasl;fjals;jdflkndgouwhetopwqhjtoj", + ) + + b.ResetTimer() + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + res := toStringNotFun(testPayload) + _ = res + } +} + +func toStringNotFun(data []byte) string { + return string(data) +} diff --git a/pool/static_pool/stream.go b/pool/static_pool/stream.go new file mode 100644 index 0000000..9e6fa07 --- /dev/null +++ b/pool/static_pool/stream.go @@ -0,0 +1,31 @@ +package static_pool //nolint:stylecheck + +import "github.com/roadrunner-server/pool/payload" + +type PExec struct { + pld *payload.Payload + err error +} + +func newPExec(pld *payload.Payload, err error) *PExec { + return &PExec{ + pld: pld, + err: err, + } +} + +func (p *PExec) Payload() *payload.Payload { + return p.pld +} + +func (p *PExec) Body() []byte { + return p.pld.Body +} + +func (p *PExec) Context() []byte { + return p.pld.Context +} + +func (p *PExec) Error() error { + return p.err +} diff --git a/pool/static_pool/supervisor.go b/pool/static_pool/supervisor.go new file mode 100644 index 0000000..4a04e28 --- /dev/null +++ b/pool/static_pool/supervisor.go @@ -0,0 +1,180 @@ +package static_pool //nolint:stylecheck + +import ( + "time" + + "github.com/roadrunner-server/events" + "github.com/roadrunner-server/pool/fsm" + "github.com/roadrunner-server/pool/state/process" + "go.uber.org/zap" +) + +const ( + MB = 1024 * 1024 + + // NsecInSec nanoseconds in second + NsecInSec int64 = 1000000000 +) + +func (sp *Pool) Start() { + go func() { + watchTout := time.NewTicker(sp.cfg.Supervisor.WatchTick) + defer watchTout.Stop() + + for { + select { + case <-sp.stopCh: + return + // stop here + case <-watchTout.C: + sp.mu.Lock() + sp.control() + sp.mu.Unlock() + } + } + }() +} + +func (sp *Pool) Stop() { + sp.stopCh <- struct{}{} +} + +func (sp *Pool) control() { + now := time.Now() + + // MIGHT BE OUTDATED + // It's a copy of the Workers pointers + workers := sp.Workers() + + for i := 0; i < len(workers); i++ { + // if worker not in the Ready OR working state + // skip such worker + switch workers[i].State().CurrentState() { + case + fsm.StateInactive, + fsm.StateErrored, + fsm.StateStopping, + fsm.StateStopped, + fsm.StateInvalid, + fsm.StateMaxJobsReached: + + // do no touch the bad worker until it pushed back to the stack + continue + + case + fsm.StateMaxMemoryReached, + fsm.StateIdleTTLReached, + fsm.StateTTLReached: + // we can stop workers which reached the idlettl state + // workers can be moved from these states ONLY by the supervisor and ONLY if the worker is in the StateReady + if workers[i] != nil { + _ = workers[i].Stop() + } + + // call cleanup callback + workers[i].Callback() + + continue + } + + s, err := process.WorkerProcessState(workers[i]) + if err != nil { + // worker not longer valid for supervision + continue + } + + if sp.cfg.Supervisor.TTL != 0 && now.Sub(workers[i].Created()).Seconds() >= sp.cfg.Supervisor.TTL.Seconds() { + /* + worker at this point might be in the middle of request execution: + + ---> REQ ---> WORKER -----------------> RESP (at this point we should not set the Ready state) ------> | ----> Worker gets between supervisor checks and get killed in the ww.Release + ^ + TTL Reached, state - invalid | + -----> Worker Stopped here + */ + + // if the worker in the StateReady, it means, that it's not working on the request and we can safely stop/kill it + // but if the worker in the any other state, we can't stop it, because it might be in the middle of the request execution, instead, we're setting the Invalid state + if workers[i].State().Compare(fsm.StateReady) { + workers[i].State().Transition(fsm.StateTTLReached) + } else { + workers[i].State().Transition(fsm.StateInvalid) + } + + sp.log.Debug("ttl", zap.String("reason", "ttl is reached"), zap.Int64("pid", workers[i].Pid()), zap.String("internal_event_name", events.EventTTL.String())) + continue + } + + if sp.cfg.Supervisor.MaxWorkerMemory != 0 && s.MemoryUsage >= sp.cfg.Supervisor.MaxWorkerMemory*MB { + /* + worker at this point might be in the middle of request execution: + + ---> REQ ---> WORKER -----------------> RESP (at this point we should not set the Ready state) ------> | ----> Worker gets between supervisor checks and get killed in the ww.Release + ^ + TTL Reached, state - invalid | + -----> Worker Stopped here + */ + + // if the worker in the StateReady, it means, that it's not working on the request and we can safely stop/kill it + // but if the worker in the any other state, we can't stop it, because it might be in the middle of the request execution, instead, we're setting the Invalid state + if workers[i].State().Compare(fsm.StateReady) { + workers[i].State().Transition(fsm.StateMaxMemoryReached) + } else { + workers[i].State().Transition(fsm.StateInvalid) + } + + sp.log.Debug("memory_limit", zap.String("reason", "max memory is reached"), zap.Int64("pid", workers[i].Pid()), zap.String("internal_event_name", events.EventMaxMemory.String())) + continue + } + + // firs we check maxWorker idle + if sp.cfg.Supervisor.IdleTTL != 0 { + // then check for the worker state + if !workers[i].State().Compare(fsm.StateReady) { + continue + } + + /* + Calculate idle time + If worker in the StateReady, we read it LastUsed timestamp as UnixNano uint64 + 2. For example maxWorkerIdle is equal to 5sec, then, if (time.Now - LastUsed) > maxWorkerIdle + we are guessing that worker overlap idle time and has to be killed + */ + + // 1610530005534416045 lu + // lu - now = -7811150814 - nanoseconds + // 7.8 seconds + // get last used unix nano + lu := workers[i].State().LastUsed() + // worker not used, skip + if lu == 0 { + continue + } + + // convert last used to unixNano and sub time.now to seconds + // negative number, because lu always in the past, except for the `back to the future` :) + res := ((int64(lu) - now.UnixNano()) / NsecInSec) * -1 + + // maxWorkerIdle more than diff between now and last used + // for example: + // After exec worker goes to the rest + // And resting for the 5 seconds + // IdleTTL is 1 second. + // After the control check, res will be 5, idle is 1 + // 5 - 1 = 4, more than 0, YOU ARE FIRED (removed). Done. + if int64(sp.cfg.Supervisor.IdleTTL.Seconds())-res <= 0 { + /* + worker at this point might be in the middle of request execution: + + ---> REQ ---> WORKER -----------------> RESP (at this point we should not set the Ready state) ------> | ----> Worker gets between supervisor checks and get killed in the ww.Release + ^ + TTL Reached, state - invalid | + -----> Worker Stopped here + */ + + workers[i].State().Transition(fsm.StateIdleTTLReached) + sp.log.Debug("idle_ttl", zap.String("reason", "idle ttl is reached"), zap.Int64("pid", workers[i].Pid()), zap.String("internal_event_name", events.EventTTL.String())) + } + } + } +} diff --git a/pool/static_pool/supervisor_test.go b/pool/static_pool/supervisor_test.go new file mode 100644 index 0000000..6b59561 --- /dev/null +++ b/pool/static_pool/supervisor_test.go @@ -0,0 +1,720 @@ +package static_pool //nolint:stylecheck + +import ( + "context" + "os" + "os/exec" + "testing" + "time" + + "github.com/roadrunner-server/pool/fsm" + "github.com/roadrunner-server/pool/ipc/pipe" + "github.com/roadrunner-server/pool/payload" + "github.com/roadrunner-server/pool/pool" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +var cfgSupervised = &pool.Config{ + NumWorkers: uint64(1), + AllocateTimeout: time.Second * 10, + DestroyTimeout: time.Second * 10, + Supervisor: &pool.SupervisorConfig{ + WatchTick: 1 * time.Second, + TTL: 100 * time.Second, + IdleTTL: 100 * time.Second, + ExecTTL: 100 * time.Second, + MaxWorkerMemory: 100, + }, +} + +func Test_SupervisedPool_Exec(t *testing.T) { + ctx := context.Background() + p, err := NewPool( + ctx, + func(cmd []string) *exec.Cmd { return exec.Command("php", "../../tests/memleak.php", "pipes") }, + pipe.NewPipeFactory(log()), + cfgSupervised, + log(), + ) + + require.NoError(t, err) + require.NotNil(t, p) + + time.Sleep(time.Second) + + pidBefore := p.Workers()[0].Pid() + + for i := 0; i < 10; i++ { + time.Sleep(time.Second) + _, err = p.Exec(ctx, &payload.Payload{ + Context: []byte(""), + Body: []byte("foo"), + }, make(chan struct{})) + require.NoError(t, err) + } + + time.Sleep(time.Second) + require.NotEqual(t, pidBefore, p.Workers()[0].Pid()) + + ctxNew, cancel := context.WithTimeout(ctx, time.Second) + p.Destroy(ctxNew) + cancel() +} + +func Test_SupervisedPool_AddRemoveWorkers(t *testing.T) { + ctx := context.Background() + p, err := NewPool( + ctx, + func(cmd []string) *exec.Cmd { return exec.Command("php", "../../tests/memleak.php", "pipes") }, + pipe.NewPipeFactory(log()), + cfgSupervised, + log(), + ) + + require.NoError(t, err) + require.NotNil(t, p) + + time.Sleep(time.Second) + + pidBefore := p.Workers()[0].Pid() + + for i := 0; i < 10; i++ { + time.Sleep(time.Second) + _, err = p.Exec(ctx, &payload.Payload{ + Context: []byte(""), + Body: []byte("foo"), + }, make(chan struct{})) + require.NoError(t, err) + } + + time.Sleep(time.Second) + require.NotEqual(t, pidBefore, p.Workers()[0].Pid()) + + ctxNew, cancel := context.WithTimeout(ctx, time.Second) + p.Destroy(ctxNew) + cancel() +} + +func Test_SupervisedPool_ImmediateDestroy(t *testing.T) { + ctx := context.Background() + + p, err := NewPool( + ctx, + func(cmd []string) *exec.Cmd { return exec.Command("php", "../../tests/client.php", "echo", "pipes") }, + pipe.NewPipeFactory(log()), + &pool.Config{ + AllocateTimeout: time.Second * 10, + DestroyTimeout: time.Second * 10, + Supervisor: &pool.SupervisorConfig{ + WatchTick: 1 * time.Second, + TTL: 100 * time.Second, + IdleTTL: 100 * time.Second, + ExecTTL: 100 * time.Second, + MaxWorkerMemory: 100, + }, + }, + log(), + ) + assert.NoError(t, err) + assert.NotNil(t, p) + + _, _ = p.Exec(ctx, &payload.Payload{Body: []byte("hello"), Context: nil}, make(chan struct{})) + + ctx, cancel := context.WithTimeout(ctx, time.Nanosecond) + defer cancel() + + p.Destroy(ctx) +} + +func Test_SupervisedPool_NilFactory(t *testing.T) { + ctx := context.Background() + p, err := NewPool( + ctx, + func(cmd []string) *exec.Cmd { return exec.Command("php", "../../tests/client.php", "echo", "pipes") }, + pipe.NewPipeFactory(log()), + nil, + log(), + ) + assert.Error(t, err) + assert.Nil(t, p) +} + +func Test_SupervisedPool_NilConfig(t *testing.T) { + ctx := context.Background() + p, err := NewPool( + ctx, + func(cmd []string) *exec.Cmd { return exec.Command("php", "../../tests/client.php", "echo", "pipes") }, + nil, + cfgSupervised, + log(), + ) + assert.Error(t, err) + assert.Nil(t, p) +} + +func Test_SupervisedPool_RemoveNoWorkers(t *testing.T) { + ctx := context.Background() + + p, err := NewPool( + ctx, + func(cmd []string) *exec.Cmd { return exec.Command("php", "../../tests/client.php", "echo", "pipes") }, + pipe.NewPipeFactory(log()), + cfgSupervised, + log(), + ) + assert.NoError(t, err) + assert.NotNil(t, p) + + _, err = p.Exec(ctx, &payload.Payload{Body: []byte("hello"), Context: nil}, make(chan struct{})) + assert.NoError(t, err) + + wrks := p.Workers() + for i := 0; i < len(wrks); i++ { + assert.NoError(t, p.RemoveWorker(ctx)) + } + + assert.Len(t, p.Workers(), 1) + p.Destroy(ctx) +} + +func Test_SupervisedPool_RemoveWorker(t *testing.T) { + ctx := context.Background() + + p, err := NewPool( + ctx, + func(cmd []string) *exec.Cmd { return exec.Command("php", "../../tests/client.php", "echo", "pipes") }, + pipe.NewPipeFactory(log()), + cfgSupervised, + log(), + ) + assert.NoError(t, err) + assert.NotNil(t, p) + + _, err = p.Exec(ctx, &payload.Payload{Body: []byte("hello"), Context: nil}, make(chan struct{})) + assert.NoError(t, err) + + wrks := p.Workers() + for i := 0; i < len(wrks); i++ { + assert.NoError(t, p.RemoveWorker(ctx)) + } + + _, err = p.Exec(ctx, &payload.Payload{Body: []byte("hello"), Context: nil}, make(chan struct{})) + assert.Error(t, err) + + err = p.AddWorker() + assert.NoError(t, err) + + _, err = p.Exec(ctx, &payload.Payload{Body: []byte("hello"), Context: nil}, make(chan struct{})) + assert.NoError(t, err) + + assert.Len(t, p.Workers(), 2) + + p.Destroy(ctx) +} + +func Test_SupervisedPoolReset(t *testing.T) { + ctx := context.Background() + p, err := NewPool( + ctx, + func(cmd []string) *exec.Cmd { return exec.Command("php", "../../tests/client.php", "echo", "pipes") }, + pipe.NewPipeFactory(log()), + cfgSupervised, + log(), + ) + assert.NoError(t, err) + assert.NotNil(t, p) + + w := p.Workers() + if len(w) == 0 { + t.Fatal("should be workers inside") + } + + pid := w[0].Pid() + require.NoError(t, p.Reset(context.Background())) + + w2 := p.Workers() + if len(w2) == 0 { + t.Fatal("should be workers inside") + } + + require.NotEqual(t, pid, w2[0].Pid()) + p.Destroy(ctx) +} + +// This test should finish without freezes +func TestSupervisedPool_ExecWithDebugMode(t *testing.T) { + ctx := context.Background() + p, err := NewPool( + ctx, + func(cmd []string) *exec.Cmd { return exec.Command("php", "../../tests/supervised.php") }, + pipe.NewPipeFactory(log()), + &pool.Config{ + Debug: true, + NumWorkers: uint64(1), + AllocateTimeout: time.Second, + DestroyTimeout: time.Second, + Supervisor: &pool.SupervisorConfig{ + WatchTick: 1 * time.Second, + TTL: 100 * time.Second, + IdleTTL: 100 * time.Second, + ExecTTL: 100 * time.Second, + MaxWorkerMemory: 100, + }, + }, + log(), + ) + + assert.NoError(t, err) + assert.NotNil(t, p) + + time.Sleep(time.Second) + + for i := 0; i < 10; i++ { + time.Sleep(time.Second) + _, err = p.Exec(ctx, &payload.Payload{ + Context: []byte(""), + Body: []byte("foo"), + }, make(chan struct{})) + assert.NoError(t, err) + } + + p.Destroy(context.Background()) +} + +func TestSupervisedPool_ExecTTL_TimedOut(t *testing.T) { + var cfgExecTTL = &pool.Config{ + NumWorkers: uint64(1), + AllocateTimeout: time.Second, + DestroyTimeout: time.Second, + Supervisor: &pool.SupervisorConfig{ + WatchTick: 1 * time.Second, + TTL: 100 * time.Second, + IdleTTL: 100 * time.Second, + ExecTTL: 1 * time.Second, + MaxWorkerMemory: 100, + }, + } + ctx := context.Background() + p, err := NewPool( + ctx, + func(cmd []string) *exec.Cmd { return exec.Command("php", "../../tests/sleep.php", "pipes") }, + pipe.NewPipeFactory(log()), + cfgExecTTL, + log(), + ) + + assert.NoError(t, err) + assert.NotNil(t, p) + defer p.Destroy(context.Background()) + + pid := p.Workers()[0].Pid() + + _, err = p.Exec(ctx, &payload.Payload{ + Context: []byte(""), + Body: []byte("foo"), + }, make(chan struct{})) + assert.Error(t, err) + + time.Sleep(time.Second * 1) + // should be new worker with new pid + assert.NotEqual(t, pid, p.Workers()[0].Pid()) +} + +func TestSupervisedPool_TTL_WorkerRestarted(t *testing.T) { + var cfgExecTTL = &pool.Config{ + NumWorkers: uint64(1), + Supervisor: &pool.SupervisorConfig{ + WatchTick: 1 * time.Second, + TTL: 5 * time.Second, + }, + } + ctx := context.Background() + p, err := NewPool( + ctx, + func(cmd []string) *exec.Cmd { return exec.Command("php", "../../tests/sleep-ttl.php") }, + pipe.NewPipeFactory(log()), + cfgExecTTL, + log(), + ) + + assert.NoError(t, err) + assert.NotNil(t, p) + + pid := p.Workers()[0].Pid() + + respCh, err := p.Exec(ctx, &payload.Payload{ + Context: []byte(""), + Body: []byte("foo"), + }, make(chan struct{})) + assert.NoError(t, err) + + resp := <-respCh + + assert.Equal(t, string(resp.Body()), "hello world") + assert.Empty(t, resp.Context()) + + time.Sleep(time.Second) + assert.NotEqual(t, pid, p.Workers()[0].Pid()) + require.Equal(t, p.Workers()[0].State().CurrentState(), fsm.StateReady) + pid = p.Workers()[0].Pid() + + respCh, err = p.Exec(ctx, &payload.Payload{ + Context: []byte(""), + Body: []byte("foo"), + }, make(chan struct{})) + assert.NoError(t, err) + + resp = <-respCh + + assert.Equal(t, string(resp.Body()), "hello world") + assert.Empty(t, resp.Context()) + + time.Sleep(time.Second) + // should be new worker with new pid + assert.NotEqual(t, pid, p.Workers()[0].Pid()) + require.Equal(t, p.Workers()[0].State().CurrentState(), fsm.StateReady) + + p.Destroy(context.Background()) +} + +func TestSupervisedPool_Idle(t *testing.T) { + var cfgExecTTL = &pool.Config{ + NumWorkers: uint64(1), + AllocateTimeout: time.Second, + DestroyTimeout: time.Second, + Supervisor: &pool.SupervisorConfig{ + WatchTick: 1 * time.Second, + TTL: 100 * time.Second, + IdleTTL: 1 * time.Second, + ExecTTL: 100 * time.Second, + MaxWorkerMemory: 100, + }, + } + ctx := context.Background() + p, err := NewPool( + ctx, + func(cmd []string) *exec.Cmd { return exec.Command("php", "../../tests/idle.php", "pipes") }, + pipe.NewPipeFactory(log()), + cfgExecTTL, + log(), + ) + + assert.NoError(t, err) + assert.NotNil(t, p) + + pid := p.Workers()[0].Pid() + + respCh, err := p.Exec(ctx, &payload.Payload{ + Context: []byte(""), + Body: []byte("foo"), + }, make(chan struct{})) + assert.NoError(t, err) + + resp := <-respCh + + assert.Empty(t, resp.Body()) + assert.Empty(t, resp.Context()) + + time.Sleep(time.Second * 5) + + // worker should be marked as invalid and reallocated + _, err = p.Exec(ctx, &payload.Payload{ + Context: []byte(""), + Body: []byte("foo"), + }, make(chan struct{})) + + assert.NoError(t, err) + time.Sleep(time.Second * 2) + require.Len(t, p.Workers(), 1) + // should be new worker with new pid + assert.NotEqual(t, pid, p.Workers()[0].Pid()) + p.Destroy(context.Background()) +} + +func TestSupervisedPool_IdleTTL_StateAfterTimeout(t *testing.T) { + var cfgExecTTL = &pool.Config{ + NumWorkers: uint64(1), + AllocateTimeout: time.Second, + DestroyTimeout: time.Second, + Supervisor: &pool.SupervisorConfig{ + WatchTick: 1 * time.Second, + IdleTTL: 1 * time.Second, + MaxWorkerMemory: 100, + }, + } + ctx := context.Background() + p, err := NewPool( + ctx, + func(cmd []string) *exec.Cmd { return exec.Command("php", "../../tests/exec_ttl.php", "pipes") }, + pipe.NewPipeFactory(log()), + cfgExecTTL, + log(), + ) + + assert.NoError(t, err) + assert.NotNil(t, p) + + pid := p.Workers()[0].Pid() + + time.Sleep(time.Millisecond * 100) + respCh, err := p.Exec(ctx, &payload.Payload{ + Context: []byte(""), + Body: []byte("foo"), + }, make(chan struct{})) + assert.NoError(t, err) + + resp := <-respCh + + assert.Empty(t, resp.Body()) + assert.Empty(t, resp.Context()) + + time.Sleep(time.Second * 5) + + if len(p.Workers()) < 1 { + t.Fatal("should be at least 1 worker") + return + } + + // should be destroyed, state should be Ready, not Invalid + assert.NotEqual(t, pid, p.Workers()[0].Pid()) + assert.Equal(t, fsm.StateReady, p.Workers()[0].State().CurrentState()) + p.Destroy(context.Background()) +} + +func TestSupervisedPool_ExecTTL_OK(t *testing.T) { + var cfgExecTTL = &pool.Config{ + NumWorkers: uint64(1), + AllocateTimeout: time.Second, + DestroyTimeout: time.Second, + Supervisor: &pool.SupervisorConfig{ + WatchTick: 1 * time.Second, + TTL: 100 * time.Second, + IdleTTL: 100 * time.Second, + ExecTTL: 4 * time.Second, + MaxWorkerMemory: 100, + }, + } + ctx := context.Background() + p, err := NewPool( + ctx, + func(cmd []string) *exec.Cmd { return exec.Command("php", "../../tests/exec_ttl.php", "pipes") }, + pipe.NewPipeFactory(log()), + cfgExecTTL, + log(), + ) + + assert.NoError(t, err) + assert.NotNil(t, p) + defer p.Destroy(context.Background()) + + pid := p.Workers()[0].Pid() + + time.Sleep(time.Millisecond * 100) + respCh, err := p.Exec(ctx, &payload.Payload{ + Context: []byte(""), + Body: []byte("foo"), + }, make(chan struct{})) + assert.NoError(t, err) + + resp := <-respCh + + assert.Empty(t, resp.Body()) + assert.Empty(t, resp.Context()) + + time.Sleep(time.Second * 1) + // should be the same pid + assert.Equal(t, pid, p.Workers()[0].Pid()) +} + +func TestSupervisedPool_ShouldRespond(t *testing.T) { + var cfgExecTTL = &pool.Config{ + NumWorkers: uint64(1), + AllocateTimeout: time.Minute, + DestroyTimeout: time.Minute, + Supervisor: &pool.SupervisorConfig{ + ExecTTL: 90 * time.Second, + MaxWorkerMemory: 100, + }, + } + + // constructed + // max memory + // constructed + ctx := context.Background() + p, err := NewPool( + ctx, + func(cmd []string) *exec.Cmd { + return exec.Command("php", "../../tests/should-not-be-killed.php", "pipes") + }, + pipe.NewPipeFactory(log()), + cfgExecTTL, + log(), + ) + + assert.NoError(t, err) + assert.NotNil(t, p) + + respCh, err := p.Exec(ctx, &payload.Payload{ + Context: []byte(""), + Body: []byte("foo"), + }, make(chan struct{})) + assert.NoError(t, err) + + resp := <-respCh + + assert.Equal(t, []byte("alive"), resp.Body()) + assert.Empty(t, resp.Context()) + + time.Sleep(time.Second) + p.Destroy(context.Background()) +} + +func TestSupervisedPool_MaxMemoryReached(t *testing.T) { + var cfgExecTTL = &pool.Config{ + NumWorkers: uint64(1), + AllocateTimeout: time.Second, + DestroyTimeout: time.Second, + Supervisor: &pool.SupervisorConfig{ + WatchTick: 1 * time.Second, + TTL: 100 * time.Second, + IdleTTL: 100 * time.Second, + ExecTTL: 4 * time.Second, + MaxWorkerMemory: 1, + }, + } + + // constructed + // max memory + // constructed + ctx := context.Background() + p, err := NewPool( + ctx, + func(cmd []string) *exec.Cmd { return exec.Command("php", "../../tests/memleak.php", "pipes") }, + pipe.NewPipeFactory(log()), + cfgExecTTL, + log(), + ) + + assert.NoError(t, err) + assert.NotNil(t, p) + + respCh, err := p.Exec(ctx, &payload.Payload{ + Context: []byte(""), + Body: []byte("foo"), + }, make(chan struct{})) + assert.NoError(t, err) + + resp := <-respCh + + assert.Empty(t, resp.Body()) + assert.Empty(t, resp.Context()) + + time.Sleep(time.Second) + p.Destroy(context.Background()) +} + +func Test_SupervisedPool_FastCancel(t *testing.T) { + ctx := context.Background() + p, err := NewPool( + ctx, + func(cmd []string) *exec.Cmd { return exec.Command("php", "../../tests/sleep.php") }, + pipe.NewPipeFactory(log()), + cfgSupervised, + log(), + ) + assert.NoError(t, err) + defer p.Destroy(ctx) + + assert.NotNil(t, p) + + newCtx, cancel := context.WithTimeout(ctx, time.Second) + defer cancel() + _, err = p.Exec(newCtx, &payload.Payload{Body: []byte("hello")}, make(chan struct{})) + assert.Error(t, err) + assert.Contains(t, err.Error(), "context deadline exceeded") +} + +func Test_SupervisedPool_AllocateFailedOK(t *testing.T) { + var cfgExecTTL = &pool.Config{ + NumWorkers: uint64(2), + AllocateTimeout: time.Second * 15, + DestroyTimeout: time.Second * 5, + Supervisor: &pool.SupervisorConfig{ + WatchTick: 1 * time.Second, + TTL: 5 * time.Second, + }, + } + + ctx := context.Background() + p, err := NewPool( + ctx, + func(cmd []string) *exec.Cmd { return exec.Command("php", "../../tests/allocate-failed.php") }, + pipe.NewPipeFactory(log()), + cfgExecTTL, + log(), + ) + + assert.NoError(t, err) + require.NotNil(t, p) + + time.Sleep(time.Second) + + // should be ok + _, err = p.Exec(ctx, &payload.Payload{ + Context: []byte(""), + Body: []byte("foo"), + }, make(chan struct{})) + require.NoError(t, err) + + // after creating this file, PHP will fail + file, err := os.Create("break") + require.NoError(t, err) + + time.Sleep(time.Second * 5) + assert.NoError(t, file.Close()) + assert.NoError(t, os.Remove("break")) + + defer func() { + if r := recover(); r != nil { + assert.Fail(t, "panic should not be fired!") + } else { + p.Destroy(context.Background()) + } + }() +} + +func Test_SupervisedPool_NoFreeWorkers(t *testing.T) { + ctx := context.Background() + + p, err := NewPool( + ctx, + // sleep for the 3 seconds + func(cmd []string) *exec.Cmd { return exec.Command("php", "../../tests/sleep.php", "pipes") }, + pipe.NewPipeFactory(log()), + &pool.Config{ + Debug: false, + NumWorkers: 1, + AllocateTimeout: time.Second, + DestroyTimeout: time.Second, + Supervisor: &pool.SupervisorConfig{}, + }, + log(), + ) + assert.NoError(t, err) + assert.NotNil(t, p) + + go func() { + ctxNew, cancel := context.WithTimeout(ctx, time.Second*5) + defer cancel() + _, _ = p.Exec(ctxNew, &payload.Payload{Body: []byte("hello")}, make(chan struct{})) + }() + + time.Sleep(time.Second) + _, err = p.Exec(ctx, &payload.Payload{Body: []byte("hello")}, make(chan struct{})) + assert.Error(t, err) + + time.Sleep(time.Second) + + p.Destroy(ctx) +} diff --git a/process/isolate.go b/process/isolate.go new file mode 100755 index 0000000..8e44351 --- /dev/null +++ b/process/isolate.go @@ -0,0 +1,59 @@ +//go:build !windows + +package process + +import ( + "fmt" + "os" + "os/exec" + "os/user" + "strconv" + "syscall" + + "github.com/roadrunner-server/errors" +) + +// IsolateProcess change gpid for the process to avoid bypassing signals to php processes. +func IsolateProcess(cmd *exec.Cmd) { + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true, Pgid: 0} +} + +// ExecuteFromUser may work only if run RR under root user +func ExecuteFromUser(cmd *exec.Cmd, u string) error { + const op = errors.Op("execute_from_user") + usr, err := user.Lookup(u) + if err != nil { + return errors.E(op, err) + } + + usrI32, err := strconv.ParseInt(usr.Uid, 10, 32) + if err != nil { + return errors.E(op, err) + } + + grI32, err := strconv.ParseInt(usr.Gid, 10, 32) + if err != nil { + return errors.E(op, err) + } + + // For more information: + // https://www.man7.org/linux/man-pages/man7/user_namespaces.7.html + // https://www.man7.org/linux/man-pages/man7/namespaces.7.html + if _, err := os.Stat("/proc/self/ns/user"); err != nil { + if os.IsNotExist(err) { + return fmt.Errorf("kernel doesn't support user namespaces") + } + if os.IsPermission(err) { + return fmt.Errorf("unable to test user namespaces due to permissions") + } + + return errors.E(op, errors.Errorf("failed to stat /proc/self/ns/user: %v", err)) + } + + cmd.SysProcAttr.Credential = &syscall.Credential{ + Uid: uint32(usrI32), + Gid: uint32(grI32), + } + + return nil +} diff --git a/process/isolate_windows.go b/process/isolate_windows.go new file mode 100755 index 0000000..df977c1 --- /dev/null +++ b/process/isolate_windows.go @@ -0,0 +1,17 @@ +//go:build windows + +package process + +import ( + "os/exec" + "syscall" +) + +// IsolateProcess change gpid for the process to avoid bypassing signals to php processes. +func IsolateProcess(cmd *exec.Cmd) { + cmd.SysProcAttr = &syscall.SysProcAttr{CreationFlags: syscall.CREATE_NEW_PROCESS_GROUP} +} + +func ExecuteFromUser(cmd *exec.Cmd, u string) error { + return nil +} diff --git a/state/process/state.go b/state/process/state.go new file mode 100644 index 0000000..039fe50 --- /dev/null +++ b/state/process/state.go @@ -0,0 +1,53 @@ +package process + +import ( + "github.com/roadrunner-server/errors" + "github.com/roadrunner-server/pool/worker" + "github.com/shirou/gopsutil/process" +) + +// State provides information about specific worker. +type State struct { + // Pid contains process id. + Pid int64 `json:"pid"` + // Status of the worker. + Status int64 `json:"status"` + // Number of worker executions. + NumExecs uint64 `json:"numExecs"` + // Created is unix nano timestamp of worker creation time. + Created int64 `json:"created"` + // MemoryUsage holds the information about worker memory usage in bytes. + // Values might vary for different operating systems and based on RSS. + MemoryUsage uint64 `json:"memoryUsage"` + // CPU_Percent returns how many percent of the CPU time this process uses + CPUPercent float64 `json:"CPUPercent"` + // Command used in the service plugin and shows a command for the particular service + Command string `json:"command"` + // Status + StatusStr string `json:"statusStr"` +} + +// WorkerProcessState creates new worker state definition. +func WorkerProcessState(w *worker.Process) (*State, error) { + const op = errors.Op("worker_process_state") + p, _ := process.NewProcess(int32(w.Pid())) + i, err := p.MemoryInfo() + if err != nil { + return nil, errors.E(op, err) + } + + percent, err := p.CPUPercent() + if err != nil { + return nil, err + } + + return &State{ + CPUPercent: percent, + Pid: w.Pid(), + Status: w.State().CurrentState(), + StatusStr: w.State().String(), + NumExecs: w.State().NumExecs(), + Created: w.Created().UnixNano(), + MemoryUsage: i.RSS, + }, nil +} diff --git a/tests/allocate-failed.php b/tests/allocate-failed.php new file mode 100644 index 0000000..8514ecc --- /dev/null +++ b/tests/allocate-failed.php @@ -0,0 +1,18 @@ +waitPayload()){ + $rr->respond(new \Spiral\RoadRunner\Payload("")); +} diff --git a/tests/broken.php b/tests/broken.php new file mode 100644 index 0000000..413f860 --- /dev/null +++ b/tests/broken.php @@ -0,0 +1,14 @@ +waitPayload()) { + echo undefined_function(); + $rr->respond(new RoadRunner\Payload((string)$in->body, null)); +} diff --git a/tests/client.php b/tests/client.php new file mode 100644 index 0000000..f81d33e --- /dev/null +++ b/tests/client.php @@ -0,0 +1,35 @@ +=1.0", + "spiral/tokenizer": ">=2.7", + "spiral/goridge": "^3.2", + "spiral/roadrunner-metrics": "^2.0" + }, + "autoload": { + "psr-4": { + "Temporal\\Tests\\": "src" + } + }, + "name": "test/test", + "description": "test" +} diff --git a/tests/crc_error.php b/tests/crc_error.php new file mode 100644 index 0000000..a769fb1 --- /dev/null +++ b/tests/crc_error.php @@ -0,0 +1,17 @@ +waitPayload()){ + sleep(3); + $rr->respond(new \Spiral\RoadRunner\Payload("")); +} diff --git a/tests/delay.php b/tests/delay.php new file mode 100644 index 0000000..2c8255b --- /dev/null +++ b/tests/delay.php @@ -0,0 +1,18 @@ +waitPayload()) { + try { + usleep($in->body * 1000); + $rr->respond(new RoadRunner\Payload('')); + } catch (\Throwable $e) { + $rr->error((string)$e); + } +} diff --git a/tests/echo.php b/tests/echo.php new file mode 100644 index 0000000..6451046 --- /dev/null +++ b/tests/echo.php @@ -0,0 +1,17 @@ +waitPayload()) { + try { + $rr->respond(new RoadRunner\Payload((string)$in->body)); + } catch (\Throwable $e) { + $rr->error((string)$e); + } +} diff --git a/tests/error.php b/tests/error.php new file mode 100644 index 0000000..c77e681 --- /dev/null +++ b/tests/error.php @@ -0,0 +1,13 @@ +waitPayload()) { + $rr->error((string)$in->body); +} diff --git a/tests/exec_ttl.php b/tests/exec_ttl.php new file mode 100644 index 0000000..fb5c9df --- /dev/null +++ b/tests/exec_ttl.php @@ -0,0 +1,15 @@ +waitPayload()){ + sleep(3); + $rr->respond(new \Spiral\RoadRunner\Payload("")); +} diff --git a/tests/failboot.php b/tests/failboot.php new file mode 100644 index 0000000..d59462c --- /dev/null +++ b/tests/failboot.php @@ -0,0 +1,3 @@ +waitPayload()) { + try { + $rr->respond(new RoadRunner\Payload("", (string)$in->header)); + } catch (\Throwable $e) { + $rr->error((string)$e); + } +} diff --git a/tests/http/client.php b/tests/http/client.php new file mode 100644 index 0000000..90b5c2b --- /dev/null +++ b/tests/http/client.php @@ -0,0 +1,51 @@ +waitRequest()) { + try { + $psr7->respond(handleRequest($req, new \Nyholm\Psr7\Response())); + } catch (\Throwable $e) { + $psr7->getWorker()->error((string)$e); + } +} diff --git a/tests/http/cookie.php b/tests/http/cookie.php new file mode 100644 index 0000000..97673ef --- /dev/null +++ b/tests/http/cookie.php @@ -0,0 +1,334 @@ +getBody()->write(strtoupper($req->getCookieParams()['input'])); + + return $resp->withAddedHeader( + "Set-Cookie", + (new Cookie('output', 'cookie-output'))->createHeader() + ); +} + +final class Cookie +{ + /** + * The name of the cookie. + * + * @var string + */ + private $name = ''; + /** + * The value of the cookie. This value is stored on the clients computer; do not store sensitive + * information. + * + * @var string|null + */ + private $value = null; + /** + * Cookie lifetime. This value specified in seconds and declares period of time in which cookie + * will expire relatively to current time() value. + * + * @var int|null + */ + private $lifetime = null; + /** + * The path on the server in which the cookie will be available on. + * + * If set to '/', the cookie will be available within the entire domain. If set to '/foo/', + * the cookie will only be available within the /foo/ directory and all sub-directories such as + * /foo/bar/ of domain. The default value is the current directory that the cookie is being set + * in. + * + * @var string|null + */ + private $path = null; + /** + * The domain that the cookie is available. To make the cookie available on all subdomains of + * example.com then you'd set it to '.example.com'. The . is not required but makes it + * compatible with more browsers. Setting it to www.example.com will make the cookie only + * available in the www subdomain. Refer to tail matching in the spec for details. + * + * @var string|null + */ + private $domain = null; + /** + * Indicates that the cookie should only be transmitted over a secure HTTPS connection from the + * client. When set to true, the cookie will only be set if a secure connection exists. + * On the server-side, it's on the programmer to send this kind of cookie only on secure + * connection + * (e.g. with respect to $_SERVER["HTTPS"]). + * + * @var bool|null + */ + private $secure = null; + /** + * When true the cookie will be made accessible only through the HTTP protocol. This means that + * the cookie won't be accessible by scripting languages, such as JavaScript. This setting can + * effectively help to reduce identity theft through XSS attacks (although it is not supported + * by all browsers). + * + * @var bool + */ + private $httpOnly = true; + + /** + * New Cookie instance, cookies used to schedule cookie set while dispatching Response. + * + * @link http://php.net/manual/en/function.setcookie.php + * + * @param string $name The name of the cookie. + * @param string $value The value of the cookie. This value is stored on the clients + * computer; do not store sensitive information. + * @param int $lifetime Cookie lifetime. This value specified in seconds and declares period + * of time in which cookie will expire relatively to current time() + * value. + * @param string $path The path on the server in which the cookie will be available on. + * If set to '/', the cookie will be available within the entire + * domain. + * If set to '/foo/', the cookie will only be available within the + * /foo/ + * directory and all sub-directories such as /foo/bar/ of domain. The + * default value is the current directory that the cookie is being set + * in. + * @param string $domain The domain that the cookie is available. To make the cookie + * available + * on all subdomains of example.com then you'd set it to + * '.example.com'. + * The . is not required but makes it compatible with more browsers. + * Setting it to www.example.com will make the cookie only available in + * the www subdomain. Refer to tail matching in the spec for details. + * @param bool $secure Indicates that the cookie should only be transmitted over a secure + * HTTPS connection from the client. When set to true, the cookie will + * only be set if a secure connection exists. On the server-side, it's + * on the programmer to send this kind of cookie only on secure + * connection (e.g. with respect to $_SERVER["HTTPS"]). + * @param bool $httpOnly When true the cookie will be made accessible only through the HTTP + * protocol. This means that the cookie won't be accessible by + * scripting + * languages, such as JavaScript. This setting can effectively help to + * reduce identity theft through XSS attacks (although it is not + * supported by all browsers). + */ + public function __construct( + string $name, + string $value = null, + int $lifetime = null, + string $path = null, + string $domain = null, + bool $secure = false, + bool $httpOnly = true + ) { + $this->name = $name; + $this->value = $value; + $this->lifetime = $lifetime; + $this->path = $path; + $this->domain = $domain; + $this->secure = $secure; + $this->httpOnly = $httpOnly; + } + + /** + * The name of the cookie. + * + * @return string + */ + public function getName(): string + { + return $this->name; + } + + /** + * The value of the cookie. This value is stored on the clients computer; do not store sensitive + * information. + * + * @return string|null + */ + public function getValue() + { + return $this->value; + } + + /** + * The time the cookie expires. This is a Unix timestamp so is in number of seconds since the + * epoch. In other words, you'll most likely set this with the time function plus the number of + * seconds before you want it to expire. Or you might use mktime. + * + * Will return null if lifetime is not specified. + * + * @return int|null + */ + public function getExpires() + { + if ($this->lifetime === null) { + return null; + } + + return time() + $this->lifetime; + } + + /** + * The path on the server in which the cookie will be available on. + * + * If set to '/', the cookie will be available within the entire domain. If set to '/foo/', + * the cookie will only be available within the /foo/ directory and all sub-directories such as + * /foo/bar/ of domain. The default value is the current directory that the cookie is being set + * in. + * + * @return string|null + */ + public function getPath() + { + return $this->path; + } + + /** + * The domain that the cookie is available. To make the cookie available on all subdomains of + * example.com then you'd set it to '.example.com'. The . is not required but makes it + * compatible with more browsers. Setting it to www.example.com will make the cookie only + * available in the www subdomain. Refer to tail matching in the spec for details. + * + * @return string|null + */ + public function getDomain() + { + return $this->domain; + } + + /** + * Indicates that the cookie should only be transmitted over a secure HTTPS connection from the + * client. When set to true, the cookie will only be set if a secure connection exists. + * On the server-side, it's on the programmer to send this kind of cookie only on secure + * connection + * (e.g. with respect to $_SERVER["HTTPS"]). + * + * @return bool + */ + public function isSecure(): bool + { + return $this->secure; + } + + /** + * When true the cookie will be made accessible only through the HTTP protocol. This means that + * the cookie won't be accessible by scripting languages, such as JavaScript. This setting can + * effectively help to reduce identity theft through XSS attacks (although it is not supported + * by all browsers). + * + * @return bool + */ + public function isHttpOnly(): bool + { + return $this->httpOnly; + } + + /** + * Get new cookie with altered value. Original cookie object should not be changed. + * + * @param string $value + * + * @return Cookie + */ + public function withValue(string $value): self + { + $cookie = clone $this; + $cookie->value = $value; + + return $cookie; + } + + /** + * Convert cookie instance to string. + * + * @link http://www.w3.org/Protocols/rfc2109/rfc2109 + * @return string + */ + public function createHeader(): string + { + $header = [ + rawurlencode($this->name) . '=' . rawurlencode($this->value) + ]; + if ($this->lifetime !== null) { + $header[] = 'Expires=' . gmdate(\DateTime::COOKIE, $this->getExpires()); + $header[] = 'Max-Age=' . $this->lifetime; + } + if (!empty($this->path)) { + $header[] = 'Path=' . $this->path; + } + if (!empty($this->domain)) { + $header[] = 'Domain=' . $this->domain; + } + if ($this->secure) { + $header[] = 'Secure'; + } + if ($this->httpOnly) { + $header[] = 'HttpOnly'; + } + + return join('; ', $header); + } + + /** + * New Cookie instance, cookies used to schedule cookie set while dispatching Response. + * Static constructor. + * + * @link http://php.net/manual/en/function.setcookie.php + * + * @param string $name The name of the cookie. + * @param string $value The value of the cookie. This value is stored on the clients + * computer; do not store sensitive information. + * @param int $lifetime Cookie lifetime. This value specified in seconds and declares period + * of time in which cookie will expire relatively to current time() + * value. + * @param string $path The path on the server in which the cookie will be available on. + * If set to '/', the cookie will be available within the entire + * domain. + * If set to '/foo/', the cookie will only be available within the + * /foo/ + * directory and all sub-directories such as /foo/bar/ of domain. The + * default value is the current directory that the cookie is being set + * in. + * @param string $domain The domain that the cookie is available. To make the cookie + * available + * on all subdomains of example.com then you'd set it to + * '.example.com'. + * The . is not required but makes it compatible with more browsers. + * Setting it to www.example.com will make the cookie only available in + * the www subdomain. Refer to tail matching in the spec for details. + * @param bool $secure Indicates that the cookie should only be transmitted over a secure + * HTTPS connection from the client. When set to true, the cookie will + * only be set if a secure connection exists. On the server-side, it's + * on the programmer to send this kind of cookie only on secure + * connection (e.g. with respect to $_SERVER["HTTPS"]). + * @param bool $httpOnly When true the cookie will be made accessible only through the HTTP + * protocol. This means that the cookie won't be accessible by + * scripting + * languages, such as JavaScript. This setting can effectively help to + * reduce identity theft through XSS attacks (although it is not + * supported by all browsers). + * + * @return Cookie + */ + public static function create( + string $name, + string $value = null, + int $lifetime = null, + string $path = null, + string $domain = null, + bool $secure = false, + bool $httpOnly = true + ): self { + return new self($name, $value, $lifetime, $path, $domain, $secure, $httpOnly); + } + + /** + * @return string + */ + public function __toString(): string + { + return $this->createHeader(); + } +} diff --git a/tests/http/data.php b/tests/http/data.php new file mode 100644 index 0000000..6570936 --- /dev/null +++ b/tests/http/data.php @@ -0,0 +1,17 @@ +getParsedBody(); + + ksort($data); + ksort($data['arr']); + ksort($data['arr']['x']['y']); + + $resp->getBody()->write(json_encode($data)); + + return $resp; +} diff --git a/tests/http/echo.php b/tests/http/echo.php new file mode 100644 index 0000000..08e29a2 --- /dev/null +++ b/tests/http/echo.php @@ -0,0 +1,10 @@ +getBody()->write(strtoupper($req->getQueryParams()['hello'])); + return $resp->withStatus(201); +} diff --git a/tests/http/echoDelay.php b/tests/http/echoDelay.php new file mode 100644 index 0000000..78e8547 --- /dev/null +++ b/tests/http/echoDelay.php @@ -0,0 +1,11 @@ +getBody()->write(strtoupper($req->getQueryParams()['hello'])); + return $resp->withStatus(201); +} diff --git a/tests/http/echoerr.php b/tests/http/echoerr.php new file mode 100644 index 0000000..7e1d05e --- /dev/null +++ b/tests/http/echoerr.php @@ -0,0 +1,12 @@ +getQueryParams()['hello'])); + + $resp->getBody()->write(strtoupper($req->getQueryParams()['hello'])); + return $resp->withStatus(201); +} diff --git a/tests/http/env.php b/tests/http/env.php new file mode 100644 index 0000000..3755bde --- /dev/null +++ b/tests/http/env.php @@ -0,0 +1,10 @@ +getBody()->write($_SERVER['ENV_KEY']); + return $resp; +} diff --git a/tests/http/error.php b/tests/http/error.php new file mode 100644 index 0000000..527e406 --- /dev/null +++ b/tests/http/error.php @@ -0,0 +1,9 @@ +getBody()->write(strtoupper($req->getHeaderLine('input'))); + + return $resp->withAddedHeader("Header", $req->getQueryParams()['hello']); +} diff --git a/tests/http/headers.php b/tests/http/headers.php new file mode 100644 index 0000000..b6f3967 --- /dev/null +++ b/tests/http/headers.php @@ -0,0 +1,11 @@ +getBody()->write(json_encode($req->getHeaders())); + + return $resp; +} diff --git a/tests/http/ip.php b/tests/http/ip.php new file mode 100644 index 0000000..49eb928 --- /dev/null +++ b/tests/http/ip.php @@ -0,0 +1,11 @@ +getBody()->write($req->getServerParams()['REMOTE_ADDR']); + + return $resp; +} diff --git a/tests/http/memleak.php b/tests/http/memleak.php new file mode 100644 index 0000000..197a7fb --- /dev/null +++ b/tests/http/memleak.php @@ -0,0 +1,11 @@ +getHeaderLine("Content-Type") != 'application/json') { + $resp->getBody()->write("invalid content-type"); + return $resp; + } + + // we expect json body + $p = json_decode($req->getBody(), true); + $resp->getBody()->write(json_encode(array_flip($p))); + + return $resp; +} diff --git a/tests/http/pid.php b/tests/http/pid.php new file mode 100644 index 0000000..f22d8e2 --- /dev/null +++ b/tests/http/pid.php @@ -0,0 +1,11 @@ +getBody()->write((string)getmypid()); + + return $resp; +} diff --git a/tests/http/push.php b/tests/http/push.php new file mode 100644 index 0000000..d88fc07 --- /dev/null +++ b/tests/http/push.php @@ -0,0 +1,10 @@ +getBody()->write(strtoupper($req->getQueryParams()['hello'])); + return $resp->withAddedHeader("Http2-Push", __FILE__)->withStatus(201); +} diff --git a/tests/http/request-uri.php b/tests/http/request-uri.php new file mode 100644 index 0000000..d4c8755 --- /dev/null +++ b/tests/http/request-uri.php @@ -0,0 +1,10 @@ +getBody()->write($_SERVER['REQUEST_URI']); + return $resp; +} diff --git a/tests/http/server.php b/tests/http/server.php new file mode 100644 index 0000000..393d962 --- /dev/null +++ b/tests/http/server.php @@ -0,0 +1,11 @@ +getBody()->write(strtoupper($req->getHeaderLine('input'))); + + return $resp->withAddedHeader("Header", $req->getQueryParams()['hello']); +} diff --git a/tests/http/slow-client.php b/tests/http/slow-client.php new file mode 100644 index 0000000..1eaa7bc --- /dev/null +++ b/tests/http/slow-client.php @@ -0,0 +1,52 @@ +waitRequest()) { + try { + $psr7->respond(handleRequest($req, new \Nyholm\Psr7\Response())); + } catch (\Throwable $e) { + $psr7->getWorker()->error((string) $e); + } +} diff --git a/tests/http/stuck.php b/tests/http/stuck.php new file mode 100644 index 0000000..2dea057 --- /dev/null +++ b/tests/http/stuck.php @@ -0,0 +1,11 @@ +getBody()->write(strtoupper($req->getQueryParams()['hello'])); + return $resp->withStatus(201); +} diff --git a/tests/http/upload.php b/tests/http/upload.php new file mode 100644 index 0000000..5752624 --- /dev/null +++ b/tests/http/upload.php @@ -0,0 +1,35 @@ +getUploadedFiles(); + array_walk_recursive($files, function (&$v) { + /** + * @var \Psr\Http\Message\UploadedFileInterface $v + */ + + if ($v->getError()) { + $v = [ + 'name' => $v->getClientFilename(), + 'size' => $v->getSize(), + 'mime' => $v->getClientMediaType(), + 'error' => $v->getError(), + ]; + } else { + $v = [ + 'name' => $v->getClientFilename(), + 'size' => $v->getSize(), + 'mime' => $v->getClientMediaType(), + 'error' => $v->getError(), + 'sha512' => hash('sha512', $v->getStream()->__toString()), + ]; + } + }); + + $resp->getBody()->write(json_encode($files, JSON_UNESCAPED_SLASHES)); + + return $resp; +} diff --git a/tests/http/user-agent.php b/tests/http/user-agent.php new file mode 100644 index 0000000..03d7a2c --- /dev/null +++ b/tests/http/user-agent.php @@ -0,0 +1,10 @@ +getBody()->write($_SERVER['HTTP_USER_AGENT']); + return $resp; +} diff --git a/tests/idle.php b/tests/idle.php new file mode 100644 index 0000000..fb5c9df --- /dev/null +++ b/tests/idle.php @@ -0,0 +1,15 @@ +waitPayload()){ + sleep(3); + $rr->respond(new \Spiral\RoadRunner\Payload("")); +} diff --git a/tests/issue659.php b/tests/issue659.php new file mode 100644 index 0000000..2a0e4f1 --- /dev/null +++ b/tests/issue659.php @@ -0,0 +1,21 @@ +waitRequest()) { + $psr7->getWorker()->error("test_error"); +} diff --git a/tests/memleak.php b/tests/memleak.php new file mode 100644 index 0000000..96ed500 --- /dev/null +++ b/tests/memleak.php @@ -0,0 +1,15 @@ +waitPayload()){ + $mem .= str_repeat("a", 1024*1024*10); + $rr->respond(new \Spiral\RoadRunner\Payload("")); +} diff --git a/tests/metrics-issue-571.php b/tests/metrics-issue-571.php new file mode 100644 index 0000000..947ae1f --- /dev/null +++ b/tests/metrics-issue-571.php @@ -0,0 +1,37 @@ +getRPCAddress()) +); + +$metrics->declare( + 'test', + RoadRunner\Metrics\Collector::counter()->withHelp('Test counter') +); + +while ($req = $worker->waitRequest()) { + try { + $rsp = new \Nyholm\Psr7\Response(); + $rsp->getBody()->write("hello world"); + + $metrics->add('test', 1); + + $worker->respond($rsp); + } catch (\Throwable $e) { + $worker->getWorker()->error((string)$e); + } +} diff --git a/tests/pid.php b/tests/pid.php new file mode 100644 index 0000000..962b609 --- /dev/null +++ b/tests/pid.php @@ -0,0 +1,17 @@ +waitPayload()) { + try { + $rr->respond(new RoadRunner\Payload((string)getmypid())); + } catch (\Throwable $e) { + $rr->error((string)$e); + } + } diff --git a/tests/pipes_test_script.sh b/tests/pipes_test_script.sh new file mode 100755 index 0000000..c759b0a --- /dev/null +++ b/tests/pipes_test_script.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +php ../../tests/client.php echo pipes diff --git a/tests/psr-worker-bench.php b/tests/psr-worker-bench.php new file mode 100644 index 0000000..e809f38 --- /dev/null +++ b/tests/psr-worker-bench.php @@ -0,0 +1,30 @@ +waitRequest()) { + try { + $resp = new \Nyholm\Psr7\Response(); + $resp->getBody()->write("hello world"); + + $psr7->respond($resp); + } catch (\Throwable $e) { + $psr7->getWorker()->error((string)$e); + } +} diff --git a/tests/psr-worker-post.php b/tests/psr-worker-post.php new file mode 100644 index 0000000..2f54af5 --- /dev/null +++ b/tests/psr-worker-post.php @@ -0,0 +1,30 @@ +waitRequest()) { + try { + $resp = new \Nyholm\Psr7\Response(); + $resp->getBody()->write((string) $req->getBody()); + + $psr7->respond($resp); + } catch (\Throwable $e) { + $psr7->getWorker()->error((string)$e); + } +} diff --git a/tests/psr-worker-slow.php b/tests/psr-worker-slow.php new file mode 100644 index 0000000..153dff6 --- /dev/null +++ b/tests/psr-worker-slow.php @@ -0,0 +1,29 @@ +waitRequest()) { + try { + $resp = new \Nyholm\Psr7\Response(); + sleep(mt_rand(1,20)); + $resp->getBody()->write("hello world"); + + $psr7->respond($resp); + } catch (\Throwable $e) { + $psr7->getWorker()->error((string)$e); + } +} diff --git a/tests/psr-worker.php b/tests/psr-worker.php new file mode 100644 index 0000000..db53eee --- /dev/null +++ b/tests/psr-worker.php @@ -0,0 +1,28 @@ +waitRequest()) { + try { + $resp = new \Nyholm\Psr7\Response(); + $resp->getBody()->write(str_repeat("hello world", 1000)); + + $psr7->respond($resp); + } catch (\Throwable $e) { + $psr7->getWorker()->error((string)$e); + } +} diff --git a/tests/raw-error.php b/tests/raw-error.php new file mode 100644 index 0000000..3caf46c --- /dev/null +++ b/tests/raw-error.php @@ -0,0 +1,21 @@ +waitRequest()) {} diff --git a/tests/sample.txt b/tests/sample.txt new file mode 100644 index 0000000..d64a3d9 --- /dev/null +++ b/tests/sample.txt @@ -0,0 +1 @@ +sample diff --git a/tests/should-not-be-killed.php b/tests/should-not-be-killed.php new file mode 100644 index 0000000..ba34927 --- /dev/null +++ b/tests/should-not-be-killed.php @@ -0,0 +1,19 @@ +waitPayload()){ + // Allocate some memory, this should be enough to exceed the `max_worker_memory` limit defined in + // .rr.yaml + $array = range(1, 10000000); + fprintf(STDERR, 'memory allocated: ' . memory_get_usage()); + sleep(30); + + $rr->respond(new \Spiral\RoadRunner\Payload("alive")); +} diff --git a/tests/sleep-ttl.php b/tests/sleep-ttl.php new file mode 100644 index 0000000..2230e61 --- /dev/null +++ b/tests/sleep-ttl.php @@ -0,0 +1,15 @@ +waitPayload()){ + sleep(10); + $rr->respond(new \Spiral\RoadRunner\Payload("hello world")); +} diff --git a/tests/sleep.php b/tests/sleep.php new file mode 100644 index 0000000..d36ae3e --- /dev/null +++ b/tests/sleep.php @@ -0,0 +1,15 @@ +waitPayload()){ + sleep(300); + $rr->respond(new \Spiral\RoadRunner\Payload("")); +} diff --git a/tests/sleep_short.php b/tests/sleep_short.php new file mode 100644 index 0000000..863ac74 --- /dev/null +++ b/tests/sleep_short.php @@ -0,0 +1,15 @@ +waitPayload()){ + sleep(2); + $rr->respond(new \Spiral\RoadRunner\Payload("")); +} diff --git a/tests/slow-client.php b/tests/slow-client.php new file mode 100644 index 0000000..c21b45d --- /dev/null +++ b/tests/slow-client.php @@ -0,0 +1,38 @@ +waitPayload()) { + try { + sleep(1); + $rr->respond(new RoadRunner\Payload((string)getmypid())); + } catch (\Throwable $e) { + $rr->error((string)$e); + } + } diff --git a/tests/socket_test_script.sh b/tests/socket_test_script.sh new file mode 100755 index 0000000..3948c4f --- /dev/null +++ b/tests/socket_test_script.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +php ../../tests/client.php echo tcp diff --git a/tests/src/Activity/SimpleActivity.php b/tests/src/Activity/SimpleActivity.php new file mode 100644 index 0000000..576b126 --- /dev/null +++ b/tests/src/Activity/SimpleActivity.php @@ -0,0 +1,63 @@ +", $user->name, $user->email)); + } + + #[ActivityMethod] + public function slow( + string $input + ): string { + sleep(2); + + return strtolower($input); + } + + #[ActivityMethod] + public function sha512( + Bytes $input + ): string { + return hash("sha512", ($input->getData())); + } + + public function updateRunID(WorkflowExecution $e): WorkflowExecution + { + $e->setRunId('updated'); + return $e; + } + + #[ActivityMethod] + public function fail() + { + throw new \Error("failed activity"); + } +} \ No newline at end of file diff --git a/tests/src/Client/StartNewWorkflow.php b/tests/src/Client/StartNewWorkflow.php new file mode 100644 index 0000000..67bc1d0 --- /dev/null +++ b/tests/src/Client/StartNewWorkflow.php @@ -0,0 +1,23 @@ +stub = $client->newWorkflowStub(SimpleDTOWorkflow::class); + } + + public function __invoke() + { + } +} diff --git a/tests/src/Workflow/SagaWorkflow.php b/tests/src/Workflow/SagaWorkflow.php new file mode 100644 index 0000000..e47c020 --- /dev/null +++ b/tests/src/Workflow/SagaWorkflow.php @@ -0,0 +1,54 @@ +withStartToCloseTimeout(60) + ->withRetryOptions(RetryOptions::new()->withMaximumAttempts(1)) + ); + + $saga = new Workflow\Saga(); + $saga->setParallelCompensation(true); + + try { + yield $simple->echo('test'); + $saga->addCompensation( + function () use ($simple) { + yield $simple->echo('compensate echo'); + } + ); + + yield $simple->lower('TEST'); + $saga->addCompensation( + function () use ($simple) { + yield $simple->lower('COMPENSATE LOWER'); + } + ); + + yield $simple->fail(); + } catch (\Throwable $e) { + yield $saga->compensate(); + throw $e; + } + } +} diff --git a/tests/stop.php b/tests/stop.php new file mode 100644 index 0000000..9326382 --- /dev/null +++ b/tests/stop.php @@ -0,0 +1,25 @@ +waitPayload()) { + try { + if ($used) { + // kill on second attempt + $rr->stop(); + continue; + } + + $used = true; + $rr->respond(new RoadRunner\Payload((string)getmypid())); + } catch (\Throwable $e) { + $rr->error((string)$e); + } +} diff --git a/tests/stream_worker.php b/tests/stream_worker.php new file mode 100644 index 0000000..7540ba6 --- /dev/null +++ b/tests/stream_worker.php @@ -0,0 +1,34 @@ +chunk_size = 10 * 10 * 1024; +$fp = "fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"; + +while ($worker->waitPayload()) { + try { + $resp = (new Response())->withBody(Stream::create($fp)); + $psr7->respond($resp); + } catch (\Throwable $e) { + $psr7->getWorker()->error((string)$e); + } +} + +?> \ No newline at end of file diff --git a/tests/supervised.php b/tests/supervised.php new file mode 100644 index 0000000..74f8c99 --- /dev/null +++ b/tests/supervised.php @@ -0,0 +1,14 @@ +waitPayload()){ + $rr->respond(new \Spiral\RoadRunner\Payload("")); +} diff --git a/worker/options.go b/worker/options.go new file mode 100644 index 0000000..87bb24f --- /dev/null +++ b/worker/options.go @@ -0,0 +1,49 @@ +package worker + +import ( + "crypto/rand" + "math/big" + + "go.uber.org/zap" +) + +const ( + maxExecsPercentJitter uint64 = 15 +) + +type Options func(p *Process) + +func WithLog(z *zap.Logger) Options { + return func(p *Process) { + p.log = z + } +} + +func WithMaxExecs(maxExecs uint64) Options { + return func(p *Process) { + p.maxExecs = calculateMaxExecsJitter(maxExecs, maxExecsPercentJitter, p.log) + } +} + +func calculateMaxExecsJitter(maxExecs, jitter uint64, log *zap.Logger) uint64 { + if maxExecs == 0 { + return 0 + } + + random, err := rand.Int(rand.Reader, big.NewInt(int64(jitter))) + + if err != nil { + log.Debug("jitter calculation error", zap.Error(err), zap.Uint64("jitter", jitter)) + return maxExecs + } + + percent := random.Uint64() + + if percent == 0 { + return maxExecs + } + + result := (float64(maxExecs) * float64(percent)) / 100.0 + + return maxExecs + uint64(result) +} diff --git a/worker/worker.go b/worker/worker.go new file mode 100644 index 0000000..95fc538 --- /dev/null +++ b/worker/worker.go @@ -0,0 +1,674 @@ +package worker + +import ( + "bytes" + "context" + stderr "errors" + "fmt" + "io" + "os" + "os/exec" + "runtime" + "strconv" + "strings" + "sync" + "time" + + "github.com/roadrunner-server/errors" + "github.com/roadrunner-server/goridge/v3/pkg/frame" + "github.com/roadrunner-server/goridge/v3/pkg/relay" + "github.com/roadrunner-server/pool/fsm" + "github.com/roadrunner-server/pool/internal" + "github.com/roadrunner-server/pool/payload" + "go.uber.org/zap" +) + +// Process - supervised process with api over goridge.Relay. +type Process struct { + // created indicates at what time Process has been created. + created time.Time + log *zap.Logger + + // calculated maximum value with jitter + maxExecs uint64 + + callback func() + // fsm holds information about current Process state, + // number of Process executions, buf status change time. + // publicly this object is receive-only and protected using Mutex + // and atomic counter. + fsm *fsm.Fsm + + // underlying command with associated process, command must be + // provided to Process from outside in non-started form. CmdSource + // stdErr direction will be handled by Process to aggregate error message. + cmd *exec.Cmd + + // pid of the process, points to pid of underlying process and + // can be nil while process is not started. + pid int + + fPool sync.Pool + bPool sync.Pool + chPool sync.Pool + + doneCh chan struct{} + + // communication bus with underlying process. + relay relay.Relay +} + +// internal struct to pass data between goroutines +type wexec struct { + payload *payload.Payload + err error +} + +// InitBaseWorker creates new Process over given exec.cmd. +func InitBaseWorker(cmd *exec.Cmd, options ...Options) (*Process, error) { + if cmd.Process != nil { + return nil, fmt.Errorf("can't attach to running process") + } + + w := &Process{ + created: time.Now(), + cmd: cmd, + doneCh: make(chan struct{}, 1), + + fPool: sync.Pool{ + New: func() any { + return frame.NewFrame() + }, + }, + bPool: sync.Pool{ + New: func() any { + return new(bytes.Buffer) + }, + }, + chPool: sync.Pool{ + New: func() any { + return make(chan *wexec, 1) + }, + }, + } + + // add options + for i := 0; i < len(options); i++ { + options[i](w) + } + + if w.log == nil { + z, err := zap.NewProduction() + if err != nil { + return nil, err + } + + w.log = z + } + + w.fsm = fsm.NewFSM(fsm.StateInactive, w.log) + + // set self as stderr implementation (Writer interface) + rc, err := cmd.StderrPipe() + if err != nil { + return nil, err + } + + go func() { + // https://linux.die.net/man/7/pipe + // see pipe capacity + buf := make([]byte, 65536) + errCopy := copyBuffer(w, rc, buf) + if errCopy != nil { + w.log.Debug("stderr", zap.Error(errCopy)) + } + }() + + return w, nil +} + +// Pid returns worker pid. +func (w *Process) Pid() int64 { + return int64(w.pid) +} + +func (w *Process) AddCallback(cb func()) { + w.callback = cb +} + +func (w *Process) Callback() { + if w.callback == nil { + return + } + w.callback() +} + +// Created returns time, worker was created at. +func (w *Process) Created() time.Time { + return w.created +} + +// State return receive-only Process state object, state can be used to safely access +// Process status, time when status changed and number of Process executions. +func (w *Process) State() *fsm.Fsm { + return w.fsm +} + +// AttachRelay attaches relay to the worker +func (w *Process) AttachRelay(rl relay.Relay) { + w.relay = rl +} + +// Relay returns relay attached to the worker +func (w *Process) Relay() relay.Relay { + return w.relay +} + +// String returns Process description. fmt.Stringer interface +func (w *Process) String() string { + st := w.fsm.String() + // we can safely compare pid to 0 + if w.pid != 0 { + st = st + ", pid:" + strconv.Itoa(w.pid) + } + + return fmt.Sprintf( + "(`%s` [%s], num_execs: %v)", + strings.Join(w.cmd.Args, " "), + st, + w.fsm.NumExecs(), + ) +} + +func (w *Process) Start() error { + err := w.cmd.Start() + if err != nil { + return err + } + w.pid = w.cmd.Process.Pid + return nil +} + +// Wait must be called once for each Process, call will be released once Process is +// complete and will return process error (if any), if stderr is presented it is value +// will be wrapped as WorkerError. Method will return error code if php process fails +// to find or Start the script. +func (w *Process) Wait() error { + const op = errors.Op("process_wait") + var err error + err = w.cmd.Wait() + w.doneCh <- struct{}{} + + // If worker was destroyed, just exit + if w.State().Compare(fsm.StateDestroyed) { + return nil + } + + // If state is different, and err is not nil, append it to the errors + if err != nil { + w.State().Transition(fsm.StateErrored) + err = stderr.Join(err, errors.E(op, err)) + } + + // closeRelay + // at this point according to the documentation (see cmd.Wait comment) + // if worker finishes with an error, message will be written to the stderr first + // and then process.cmd.Wait return an error + err2 := w.closeRelay() + if err2 != nil { + w.State().Transition(fsm.StateErrored) + return stderr.Join(err, errors.E(op, err2)) + } + + if w.cmd.ProcessState.Success() { + w.State().Transition(fsm.StateStopped) + } + + return err +} + +func (w *Process) StreamIter() (*payload.Payload, bool, error) { + pld, err := w.receiveFrame() + if err != nil { + return nil, false, err + } + + // PING, we should respond with PONG + if pld.Flags&frame.PING != 0 { + if err := w.sendPONG(); err != nil { + return nil, false, err + } + } + + // !=0 -> we have stream bit set, so stream is available + return pld, pld.Flags&frame.STREAM != 0, nil +} + +// StreamIter returns true if stream is available and payload +func (w *Process) StreamIterWithContext(ctx context.Context) (*payload.Payload, bool, error) { + c := w.getCh() + + go func() { + rsp, err := w.receiveFrame() + if err != nil { + c <- &wexec{ + err: err, + } + + w.log.Debug("stream iter error", zap.Int64("pid", w.Pid()), zap.Error(err)) + // trash response + rsp = nil + runtime.Goexit() + } + + c <- &wexec{ + payload: rsp, + } + }() + + select { + // exec TTL reached + case <-ctx.Done(): + // we should kill the process here to ensure that it exited + errK := w.Kill() + err := stderr.Join(errK, ctx.Err()) + // we should wait for the exit from the worker + // 'c' channel here should return an error or nil + // because the goroutine holds the payload pointer (from the sync.Pool) + <-c + w.putCh(c) + return nil, false, errors.E(errors.ExecTTL, err) + case res := <-c: + w.putCh(c) + + if res.err != nil { + return nil, false, res.err + } + + // PING, we should respond with PONG + if res.payload.Flags&frame.PING != 0 { + if err := w.sendPONG(); err != nil { + return nil, false, err + } + } + + // pld.Flags&frame.STREAM !=0 -> we have stream bit set, so stream is available + return res.payload, res.payload.Flags&frame.STREAM != 0, nil + } +} + +// StreamCancel sends stop bit to the worker +func (w *Process) StreamCancel(ctx context.Context) error { + const op = errors.Op("sync_worker_stream_cancel") + if !w.State().Compare(fsm.StateWorking) { + return errors.Errorf("worker is not in the Working state, actual state: (%s)", w.State().String()) + } + + w.log.Debug("stream was canceled, sending stop bit", zap.Int64("pid", w.Pid())) + // get a frame + fr := w.getFrame() + + fr.WriteVersion(fr.Header(), frame.Version1) + + fr.SetStopBit(fr.Header()) + fr.WriteCRC(fr.Header()) + + err := w.Relay().Send(fr) + w.State().RegisterExec() + if err != nil { + w.putFrame(fr) + return errors.E(op, errors.Network, err) + } + + w.putFrame(fr) + c := w.getCh() + + w.log.Debug("stop bit was sent, waiting for the response", zap.Int64("pid", w.Pid())) + + go func() { + for { + rsp, errrf := w.receiveFrame() + if errrf != nil { + c <- &wexec{ + err: errrf, + } + + w.log.Debug("stream cancel error", zap.Int64("pid", w.Pid()), zap.Error(errrf)) + // trash response + rsp = nil + runtime.Goexit() + } + + // stream has ended + if rsp.Flags&frame.STREAM == 0 { + w.log.Debug("stream has ended", zap.Int64("pid", w.Pid())) + c <- &wexec{} + // trash response + rsp = nil + runtime.Goexit() + } + } + }() + + select { + // exec TTL reached + case <-ctx.Done(): + errK := w.Kill() + err := stderr.Join(errK, ctx.Err()) + // we should wait for the exit from the worker + // 'c' channel here should return an error or nil + // because the goroutine holds the payload pointer (from the sync.Pool) + <-c + w.putCh(c) + return errors.E(op, errors.ExecTTL, err) + case res := <-c: + w.putCh(c) + if res.err != nil { + return res.err + } + return nil + } +} + +// Exec executes payload with TTL timeout in the context. +func (w *Process) Exec(ctx context.Context, p *payload.Payload) (*payload.Payload, error) { + const op = errors.Op("worker_exec_with_timeout") + + // worker was killed before it started to work (supervisor) + if !w.State().Compare(fsm.StateReady) { + return nil, errors.E(op, errors.Retry, errors.Errorf("Process is not ready (%s)", w.State().String())) + } + + c := w.getCh() + // set last used time + w.State().SetLastUsed(uint64(time.Now().UnixNano())) + w.State().Transition(fsm.StateWorking) + + go func() { + err := w.sendFrame(p) + if err != nil { + c <- &wexec{ + err: err, + } + runtime.Goexit() + } + + w.State().RegisterExec() + rsp, err := w.receiveFrame() + if err != nil { + c <- &wexec{ + payload: rsp, + err: err, + } + + runtime.Goexit() + } + + c <- &wexec{ + payload: rsp, + } + }() + + select { + // exec TTL reached + case <-ctx.Done(): + errK := w.Kill() + err := stderr.Join(errK, ctx.Err()) + // we should wait for the exit from the worker + // 'c' channel here should return an error or nil + // because the goroutine holds the payload pointer (from the sync.Pool) + <-c + w.putCh(c) + return nil, errors.E(op, errors.ExecTTL, err) + case res := <-c: + w.putCh(c) + if res.err != nil { + return nil, res.err + } + return res.payload, nil + } +} + +// Stop sends soft termination command to the Process and waits for process completion. +func (w *Process) Stop() error { + const op = errors.Op("process_stop") + w.fsm.Transition(fsm.StateStopping) + + go func() { + w.log.Debug("sending stop request to the worker", zap.Int("pid", w.pid)) + err := internal.SendControl(w.relay, &internal.StopCommand{Stop: true}) + if err == nil { + w.fsm.Transition(fsm.StateStopped) + } + }() + + select { + // finished, sent to the doneCh is made in the Wait() method + // If we successfully sent a stop request, Wait() method will send a struct{} to the doneCh and we're done here + // otherwise we have 10 seconds before we kill the process + case <-w.doneCh: + w.log.Debug("worker stopped", zap.Int("pid", w.pid)) + return nil + case <-time.After(time.Second * 10): + // kill process + w.log.Warn("worker doesn't respond on stop command, killing process", zap.Int64("PID", w.Pid())) + _ = w.cmd.Process.Signal(os.Kill) + w.fsm.Transition(fsm.StateStopped) + return errors.E(op, errors.Network) + } +} + +// Kill kills underlying process, make sure to call Wait() func to gather +// error log from the stderr. Does not wait for process completion! +func (w *Process) Kill() error { + w.fsm.Transition(fsm.StateStopping) + err := w.cmd.Process.Kill() + if err != nil { + return err + } + w.fsm.Transition(fsm.StateStopped) + return nil +} + +// Worker stderr +func (w *Process) Write(p []byte) (int, error) { + // unsafe to use utils.AsString + w.log.Info(string(p)) + return len(p), nil +} + +func (w *Process) MaxExecsReached() bool { + return w.maxExecs > 0 && w.State().NumExecs() >= w.maxExecs +} + +func (w *Process) MaxExecs() uint64 { + return w.maxExecs +} + +// copyBuffer is the actual implementation of Copy and CopyBuffer. +func copyBuffer(dst io.Writer, src io.Reader, buf []byte) error { + for { + nr, er := src.Read(buf) + if nr > 0 { + nw, ew := dst.Write(buf[0:nr]) + if nw < 0 || nr < nw { + nw = 0 + if ew == nil { + ew = errors.Str("invalid write result") + } + } + if ew != nil { + return ew + } + if nr != nw { + return io.ErrShortWrite + } + } + if er != nil { + if er != io.EOF { + return er + } + break + } + } + + return nil +} + +// sendFrame sends frame to the worker +func (w *Process) sendFrame(p *payload.Payload) error { + const op = errors.Op("sync_worker_send_frame") + // get a frame + fr := w.getFrame() + buf := w.get() + + // can be 0 here + fr.WriteVersion(fr.Header(), frame.Version1) + fr.WriteFlags(fr.Header(), p.Codec) + + // obtain a buffer + buf.Write(p.Context) + buf.Write(p.Body) + + // Context offset + fr.WriteOptions(fr.HeaderPtr(), uint32(len(p.Context))) + fr.WritePayloadLen(fr.Header(), uint32(buf.Len())) + fr.WritePayload(buf.Bytes()) + + fr.WriteCRC(fr.Header()) + + // return buffer + w.put(buf) + + err := w.Relay().Send(fr) + if err != nil { + w.putFrame(fr) + return errors.E(op, errors.Network, err) + } + w.putFrame(fr) + return nil +} + +func (w *Process) receiveFrame() (*payload.Payload, error) { + const op = errors.Op("sync_worker_receive_frame") + + frameR := w.getFrame() + + err := w.Relay().Receive(frameR) + if err != nil { + w.putFrame(frameR) + return nil, errors.E(op, errors.Network, err) + } + + if frameR == nil { + w.putFrame(frameR) + return nil, errors.E(op, errors.Network, errors.Str("nil frame received")) + } + + codec := frameR.ReadFlags() + + if codec&frame.ERROR != byte(0) { + // we need to copy the payload because we will put the frame back to the pool + cp := make([]byte, len(frameR.Payload())) + copy(cp, frameR.Payload()) + + w.putFrame(frameR) + return nil, errors.E(op, errors.SoftJob, errors.Str(string(cp))) + } + + options := frameR.ReadOptions(frameR.Header()) + if len(options) != 1 { + w.putFrame(frameR) + return nil, errors.E(op, errors.Decode, errors.Str("options length should be equal 1 (body offset)")) + } + + // bound check + if len(frameR.Payload()) < int(options[0]) { + // we need to copy the payload because we will put the frame back to the pool + cp := make([]byte, len(frameR.Payload())) + copy(cp, frameR.Payload()) + + w.putFrame(frameR) + return nil, errors.E(errors.Network, errors.Errorf("bad payload %s", cp)) + } + + // stream + stop -> waste + // stream + ping -> response + flags := frameR.Header()[10] + pld := &payload.Payload{ + Flags: flags, + Codec: codec, + Body: make([]byte, len(frameR.Payload()[options[0]:])), + Context: make([]byte, len(frameR.Payload()[:options[0]])), + } + + // by copying we free frame's payload slice + // we do not hold the pointer from the smaller slice to the initial (which should be in the sync.Pool) + // https://blog.golang.org/slices-intro#TOC_6. + copy(pld.Body, frameR.Payload()[options[0]:]) + copy(pld.Context, frameR.Payload()[:options[0]]) + + w.putFrame(frameR) + return pld, nil +} + +func (w *Process) sendPONG() error { + // get a frame + fr := w.getFrame() + fr.WriteVersion(fr.Header(), frame.Version1) + + fr.SetPongBit(fr.Header()) + fr.WriteCRC(fr.Header()) + + err := w.Relay().Send(fr) + w.State().RegisterExec() + if err != nil { + w.putFrame(fr) + w.State().Transition(fsm.StateErrored) + return errors.E(errors.Network, err) + } + + w.putFrame(fr) + return nil +} + +func (w *Process) closeRelay() error { + if w.relay != nil { + err := w.relay.Close() + if err != nil { + return err + } + } + return nil +} + +func (w *Process) get() *bytes.Buffer { + return w.bPool.Get().(*bytes.Buffer) +} + +func (w *Process) put(b *bytes.Buffer) { + b.Reset() + w.bPool.Put(b) +} + +func (w *Process) getFrame() *frame.Frame { + return w.fPool.Get().(*frame.Frame) +} + +func (w *Process) putFrame(f *frame.Frame) { + f.Reset() + w.fPool.Put(f) +} + +func (w *Process) getCh() chan *wexec { + return w.chPool.Get().(chan *wexec) +} + +func (w *Process) putCh(ch chan *wexec) { + // just check if the chan is not empty + select { + case <-ch: + default: + } + w.chPool.Put(ch) +} diff --git a/worker/worker_test.go b/worker/worker_test.go new file mode 100644 index 0000000..593f9e2 --- /dev/null +++ b/worker/worker_test.go @@ -0,0 +1,45 @@ +package worker + +import ( + "context" + "os/exec" + "testing" + + "github.com/roadrunner-server/pool/payload" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func Test_OnStarted(t *testing.T) { + cmd := exec.Command("php", "tests/client.php", "broken", "pipes") + assert.Nil(t, cmd.Start()) + + w, err := InitBaseWorker(cmd) + assert.Nil(t, w) + assert.NotNil(t, err) + + assert.Equal(t, "can't attach to running process", err.Error()) +} + +func Test_NotStarted_String(t *testing.T) { + cmd := exec.Command("php", "tests/client.php", "echo", "pipes") + + w, _ := InitBaseWorker(cmd) + assert.Contains(t, w.String(), "php tests/client.php echo pipes") + assert.Contains(t, w.String(), "inactive") + assert.Contains(t, w.String(), "num_execs: 0") +} + +func Test_NotStarted_Exec(t *testing.T) { + cmd := exec.Command("php", "tests/client.php", "echo", "pipes") + + w, err := InitBaseWorker(cmd) + require.NoError(t, err) + + _, err = w.Exec(context.Background(), &payload.Payload{ + Body: []byte("hello"), + }) + + assert.Error(t, err) + assert.Contains(t, err.Error(), "Process is not ready (inactive)") +} diff --git a/worker_watcher/container/channel/vec.go b/worker_watcher/container/channel/vec.go new file mode 100644 index 0000000..16556d7 --- /dev/null +++ b/worker_watcher/container/channel/vec.go @@ -0,0 +1,139 @@ +package channel + +import ( + "context" + "sync" + "sync/atomic" + "time" + + "github.com/roadrunner-server/errors" + "github.com/roadrunner-server/pool/fsm" + "github.com/roadrunner-server/pool/worker" +) + +type Vec struct { + rwm sync.RWMutex + // destroy signal + destroy uint64 + // reset signal + reset uint64 + // channel with the workers + workers chan *worker.Process +} + +func NewVector() *Vec { + vec := &Vec{ + destroy: 0, + reset: 0, + workers: make(chan *worker.Process, 1000), + } + + return vec +} + +// Push is O(1) operation +// In case of TTL and full channel O(n) worst case, where n is len of the channel +func (v *Vec) Push(w *worker.Process) { + // add remove callback + select { + case v.workers <- w: + // default select branch is only possible when dealing with TTL + // because in that case, workers in the v.workers channel can be TTL-ed and killed + // but presenting in the channel + default: + // channel is full + _ = w.Kill() + } +} + +func (v *Vec) Len() int { + return len(v.workers) +} + +func (v *Vec) Pop(ctx context.Context) (*worker.Process, error) { + // remove all workers and return + if atomic.LoadUint64(&v.destroy) == 1 { + return nil, errors.E(errors.WatcherStopped) + } + + // wait for the reset to complete + for atomic.CompareAndSwapUint64(&v.reset, 1, 1) { + select { + case <-ctx.Done(): + default: + time.Sleep(time.Millisecond * 100) + } + } + + // used only for the TTL-ed workers + v.rwm.RLock() + select { + case w := <-v.workers: + v.rwm.RUnlock() + return w, nil + case <-ctx.Done(): + v.rwm.RUnlock() + return nil, errors.E(ctx.Err(), errors.NoFreeWorkers) + } +} + +func (v *Vec) ResetDone() { + atomic.StoreUint64(&v.reset, 0) +} + +func (v *Vec) Reset() { + atomic.StoreUint64(&v.reset, 1) +} + +func (v *Vec) Destroy() { + atomic.StoreUint64(&v.destroy, 1) +} + +func (v *Vec) Remove() { + // Stop Pop operations + v.rwm.Lock() + defer v.rwm.Unlock() + + /* + we can be in the default branch by the following reasons: + 1. TTL is set with no requests during the TTL + 2. Violated Get <-> Release operation (how ??) + */ + + for i := 0; i < len(v.workers); i++ { + /* + We need to drain vector until we found a worker in the Invalid/Killing/Killed/etc states. + BUT while we are draining the vector, some worker might be reallocated and pushed into the v.workers + so, down by the code, we might have a problem when pushing the new worker to the v.workers + */ + wrk := <-v.workers + + switch wrk.State().CurrentState() { + // good states + case fsm.StateWorking, fsm.StateReady: + // put the worker back + // generally, while send and receive operations are concurrent (from the channel), channel behave + // like a FIFO, but when re-sending from the same goroutine it behaves like a FILO + select { + case v.workers <- wrk: + continue + + // all bad states are here + default: + // kill the worker from the channel + wrk.State().Transition(fsm.StateInvalid) + _ = wrk.Kill() + + continue + } + /* + Bad states are here. + */ + default: + // kill the current worker (just to be sure it's dead) + if wrk != nil { + _ = wrk.Kill() + } + } + } +} diff --git a/worker_watcher/worker_watcher.go b/worker_watcher/worker_watcher.go new file mode 100644 index 0000000..3a3777b --- /dev/null +++ b/worker_watcher/worker_watcher.go @@ -0,0 +1,431 @@ +package worker_watcher //nolint:stylecheck + +import ( + "context" + "fmt" + "sync" + "sync/atomic" + "time" + + "github.com/roadrunner-server/errors" + "github.com/roadrunner-server/events" + "github.com/roadrunner-server/pool/fsm" + "github.com/roadrunner-server/pool/worker" + "github.com/roadrunner-server/pool/worker_watcher/container/channel" + "go.uber.org/zap" +) + +// Allocator is responsible for worker allocation in the pool +type Allocator func() (*worker.Process, error) + +type WorkerWatcher struct { + sync.RWMutex + // actually don't have a lot of impl here, so interface not needed + container *channel.Vec + // used to control Destroy stage (that all workers are in the container) + numWorkers uint64 + eventBus *events.Bus + + // map with the workers pointers + workers sync.Map // map[int64]*worker.Process + + log *zap.Logger + + allocator Allocator + allocateTimeout time.Duration +} + +// NewSyncWorkerWatcher is a constructor for the Watcher +func NewSyncWorkerWatcher(allocator Allocator, log *zap.Logger, numWorkers uint64, allocateTimeout time.Duration) *WorkerWatcher { + eb, _ := events.NewEventBus() + return &WorkerWatcher{ + container: channel.NewVector(), + + log: log, + eventBus: eb, + // pass a ptr to the number of workers to avoid blocking in the TTL loop + numWorkers: numWorkers, + allocateTimeout: allocateTimeout, + workers: sync.Map{}, // make(map[int64]*worker.Process, numWorkers), + allocator: allocator, + } +} + +func (ww *WorkerWatcher) Watch(workers []*worker.Process) error { + ww.Lock() + defer ww.Unlock() + for i := 0; i < len(workers); i++ { + ii := i + ww.container.Push(workers[ii]) + // add worker to watch slice + ww.workers.Store(workers[ii].Pid(), workers[ii]) + ww.addToWatch(workers[ii]) + } + return nil +} + +func (ww *WorkerWatcher) AddWorker() error { + err := ww.Allocate() + if err != nil { + return err + } + + atomic.AddUint64(&ww.numWorkers, 1) + return nil +} + +func (ww *WorkerWatcher) RemoveWorker(ctx context.Context) error { + w, err := ww.Take(ctx) + if err != nil { + return err + } + + // can't remove the last worker + if atomic.LoadUint64(&ww.numWorkers) == 1 { + ww.log.Warn("can't remove the last worker", zap.Int64("pid", w.Pid())) + return nil + } + + // destroy and stop + w.State().Transition(fsm.StateDestroyed) + _ = w.Stop() + + atomic.AddUint64(&ww.numWorkers, ^uint64(0)) + ww.workers.Delete(w.Pid()) + + return nil +} + +// Take is not a thread safe operation +func (ww *WorkerWatcher) Take(ctx context.Context) (*worker.Process, error) { + const op = errors.Op("worker_watcher_get_free_worker") + // we need lock here to prevent Pop operation when ww in the resetting state + // thread safe operation + w, err := ww.container.Pop(ctx) + if err != nil { + if errors.Is(errors.WatcherStopped, err) { + return nil, errors.E(op, errors.WatcherStopped) + } + + return nil, errors.E(op, err) + } + + // fast path, worker not nil and in the ReadyState + if w.State().Compare(fsm.StateReady) { + return w, nil + } + + // ========================================================= + // SLOW PATH + _ = w.Kill() + // no free workers in the container or worker not in the ReadyState (TTL-ed) + // try to continuously get free one + for { + w, err = ww.container.Pop(ctx) + if err != nil { + if errors.Is(errors.WatcherStopped, err) { + return nil, errors.E(op, errors.WatcherStopped) + } + return nil, errors.E(op, err) + } + + switch w.State().CurrentState() { + // return only workers in the Ready state + // check first + case fsm.StateReady: + return w, nil + case fsm.StateWorking: // how?? + ww.container.Push(w) // put it back, let worker finish the work + continue + default: + // worker doing no work because it in the container + // so we can safely kill it (inconsistent state) + _ = w.Stop() + // try to get new worker + continue + } + } +} + +func (ww *WorkerWatcher) Allocate() error { + const op = errors.Op("worker_watcher_allocate_new") + + sw, err := ww.allocator() + if err != nil { + // log incident + ww.log.Error("allocate", zap.Error(err)) + // if no timeout, return error immediately + if ww.allocateTimeout == 0 { + return errors.E(op, errors.WorkerAllocate, err) + } + + // every second + allocateFreq := time.NewTicker(time.Millisecond * 1000) + + tt := time.After(ww.allocateTimeout) + for { + select { + case <-tt: + // reduce number of workers + atomic.AddUint64(&ww.numWorkers, ^uint64(0)) + allocateFreq.Stop() + // timeout exceed, worker can't be allocated + return errors.E(op, errors.WorkerAllocate, err) + + case <-allocateFreq.C: + sw, err = ww.allocator() + if err != nil { + // log incident + ww.log.Error("allocate retry attempt failed", zap.String("internal_event_name", events.EventWorkerError.String()), zap.Error(err)) + continue + } + + // reallocated + allocateFreq.Stop() + goto done + } + } + } + +done: + // add worker to Wait + ww.addToWatch(sw) + // add new worker to the workers slice (to get information about workers in parallel) + ww.workers.Store(sw.Pid(), sw) + // push the worker to the container + ww.Release(sw) + return nil +} + +// Release O(1) operation +func (ww *WorkerWatcher) Release(w *worker.Process) { + switch w.State().CurrentState() { + case fsm.StateReady: + ww.container.Push(w) + case + // all the possible wrong states, when we can send a stop signal + fsm.StateInactive, + fsm.StateDestroyed, + fsm.StateErrored, + fsm.StateWorking, + fsm.StateInvalid, + fsm.StateMaxMemoryReached, + fsm.StateMaxJobsReached, + fsm.StateIdleTTLReached, + fsm.StateTTLReached, + fsm.StateExecTTLReached: + + err := w.Stop() + if err != nil { + ww.log.Debug("worker release", zap.Error(err)) + } + default: + // in all other cases, we have no choice rather than kill the worker + _ = w.Kill() + } +} + +func (ww *WorkerWatcher) Reset(ctx context.Context) uint64 { + // do not release new workers + ww.container.Reset() + tt := time.NewTicker(time.Millisecond * 10) + defer tt.Stop() + for { + select { + case <-tt.C: + ww.RLock() + + // that might be one of the workers is working + // to proceed, all workers should be inside a channel + if atomic.LoadUint64(&ww.numWorkers) != uint64(ww.container.Len()) { + ww.RUnlock() + continue + } + ww.RUnlock() + // All workers at this moment are in the container + // Pop operation is blocked, push can't be done, since it's not possible to pop + ww.Lock() + + wg := &sync.WaitGroup{} + ww.workers.Range(func(key, value any) bool { + wg.Add(1) + go func(k int64, v *worker.Process) { + defer wg.Done() + v.State().Transition(fsm.StateDestroyed) + // kill the worker + _ = v.Stop() + // remove worker from the channel + v.Callback() + // delete worker from the map + ww.workers.Delete(v) + }(key.(int64), value.(*worker.Process)) + return true + }) + + wg.Wait() + ww.container.ResetDone() + + // todo: rustatian, do we need this mutex? + ww.Unlock() + + return atomic.LoadUint64(&ww.numWorkers) + case <-ctx.Done(): + // kill workers + ww.Lock() + // drain workers slice + wg := &sync.WaitGroup{} + + ww.workers.Range(func(key, value any) bool { + wg.Add(1) + go func(k int64, v *worker.Process) { + defer wg.Done() + v.State().Transition(fsm.StateDestroyed) + // stop the worker + _ = v.Stop() + // remove worker from the channel + v.Callback() + // delete worker from the map + ww.workers.Delete(v) + }(key.(int64), value.(*worker.Process)) + return true + }) + + wg.Wait() + + ww.container.ResetDone() + ww.Unlock() + + return atomic.LoadUint64(&ww.numWorkers) + } + } +} + +// Destroy all underlying container (but let them complete the task) +func (ww *WorkerWatcher) Destroy(ctx context.Context) { + ww.Lock() + // do not release new workers + ww.container.Destroy() + ww.Unlock() + + tt := time.NewTicker(time.Millisecond * 10) + // destroy container, we don't use ww mutex here, since we should be able to push worker + defer tt.Stop() + for { + select { + case <-tt.C: + ww.RLock() + // that might be one of the workers is working + if atomic.LoadUint64(&ww.numWorkers) != uint64(ww.container.Len()) { + ww.RUnlock() + continue + } + + ww.RUnlock() + // All container at this moment are in the container + // Pop operation is blocked, push can't be done, since it's not possible to pop + + ww.Lock() + // drain channel, will not actually pop, only drain a channel + _, _ = ww.container.Pop(ctx) + wg := &sync.WaitGroup{} + + ww.workers.Range(func(key, value any) bool { + wg.Add(1) + go func(k int64, v *worker.Process) { + defer wg.Done() + v.State().Transition(fsm.StateDestroyed) + // kill the worker + _ = v.Stop() + + // delete worker from the map + ww.workers.Delete(v) + }(key.(int64), value.(*worker.Process)) + return true + }) + + wg.Wait() + ww.Unlock() + return + case <-ctx.Done(): + // kill workers + ww.Lock() + wg := &sync.WaitGroup{} + ww.workers.Range(func(key, value any) bool { + wg.Add(1) + go func(k int64, v *worker.Process) { + defer wg.Done() + v.State().Transition(fsm.StateDestroyed) + // kill the worker + _ = v.Stop() + // remove worker from the channel + v.Callback() + // delete worker from the map + ww.workers.Delete(v) + }(key.(int64), value.(*worker.Process)) + return true + }) + + wg.Wait() + ww.Unlock() + return + } + } +} + +// List - this is O(n) operation, and it will return copy of the actual workers +func (ww *WorkerWatcher) List() []*worker.Process { + ww.RLock() + defer ww.RUnlock() + + if atomic.LoadUint64(&ww.numWorkers) == 0 { + return nil + } + + base := make([]*worker.Process, 0, 2) + + ww.workers.Range(func(_, value any) bool { + base = append(base, value.(*worker.Process)) + return true + }) + + return base +} + +func (ww *WorkerWatcher) wait(w *worker.Process) { + err := w.Wait() + if err != nil { + ww.log.Debug("worker stopped", zap.String("internal_event_name", events.EventWorkerWaitExit.String()), zap.Error(err)) + } + + // remove worker + ww.workers.Delete(w.Pid()) + + if w.State().Compare(fsm.StateDestroyed) { + // worker was manually destroyed, no need to replace + ww.log.Debug("worker destroyed", zap.Int64("pid", w.Pid()), zap.String("internal_event_name", events.EventWorkerDestruct.String()), zap.Error(err)) + return + } + + err = ww.Allocate() + if err != nil { + ww.log.Error("failed to allocate the worker", zap.String("internal_event_name", events.EventWorkerError.String()), zap.Error(err)) + if atomic.LoadUint64(&ww.numWorkers) == 0 { + panic("no workers available, can't run the application") + } + + return + } + + // this event used mostly for the temporal plugin + ww.eventBus.Send(events.NewEvent(events.EventWorkerStopped, "worker_watcher", fmt.Sprintf("process exited, pid: %d", w.Pid()))) +} + +func (ww *WorkerWatcher) addToWatch(wb *worker.Process) { + // this callback is used to remove the bad workers from the container + wb.AddCallback(func() { + ww.container.Remove() + }) + go func() { + ww.wait(wb) + }() +}