diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000000..ab2d4a1de5 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,4 @@ +@raviteja83 +@KaustubhKumar05 +@hdz-666 +@ygit \ No newline at end of file diff --git a/.github/CODE_OF_CONDUCT.md b/.github/CODE_OF_CONDUCT.md new file mode 100644 index 0000000000..6ba377a75a --- /dev/null +++ b/.github/CODE_OF_CONDUCT.md @@ -0,0 +1,76 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to make participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, sex characteristics, gender identity and expression, +level of experience, education, socio-economic status, nationality, personal +appearance, race, religion, or sexual identity and orientation. + +## Our Standards + +Examples of behaviour that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behaviour by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or + advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic + address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behaviour and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behaviour. + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviours that they deem inappropriate, +threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. Examples of +representing a project or community include using an official project e-mail +address, posting via an official social media account or acting as an appointed +representative at an online or offline event. Representation of a project may be +further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing or otherwise unacceptable behaviour may be +reported by contacting the 100ms team. All +complaints will be reviewed and investigated and will result in a response that +is deemed necessary and appropriate to the circumstances. The project team is +obligated to maintain confidentiality concerning the reporter of an incident. +Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, +available at + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see + diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 049d4b5ba6..a58e397fe4 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -30,5 +30,4 @@ body: - Firefox - Chrome - Safari - - Microsoft Edge - \ No newline at end of file + - Microsoft Edge \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000000..852ba8201e --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,11 @@ +blank_issues_enabled: false +contact_links: + - name: 📃 100ms Documentation + url: https://www.100ms.live/docs + about: Explore the 100ms capabilities with our Popular Guides, Demos & Blogs on 100ms Documentation page + - name: 💫 Register on 100ms Dashboard + url: https://dashboard.100ms.live/register + about: Try the gold-standard for adding live audio-video to your apps for free. + - name: 🗣️ Talk to us + url: https://www.100ms.live/contact + about: Get in touch with the 100ms team today! We are committed to helping our customers maximize the potential of our cutting-edge live video platform. Whether you need advice on how to leverage live streaming for your business, pricing information, or want to learn more about the 100ms platform, our team is here to answer all your questions and provide tailored solutions to meet your unique needs. diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml index f8e83701f3..33f595c361 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yml +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -13,5 +13,4 @@ body: label: What's the feature? description: Describe the feature, who it would help, and link to any examples from other apps. validations: - required: true - \ No newline at end of file + required: true \ No newline at end of file diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 737e45d20a..684a291339 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,6 +1,29 @@ -### Details(context, link the issue, how was the bug fixed, what does the new feature do) +# Description + +_Replace this paragraph with a description of what this PR is changing or adding, and why. Consider including before/after screenshots._ + +_List which issues are fixed by this PR. You must list at least one issue._ + +--- + +## Implementation note, gotchas, related work and Future TODOs (optional) + + - -- -### Implementation note, gotchas, related work and Future TODOs (optional) +### Pre-launch Checklist + +- [ ] The [Documentation] is updated accordingly, or this PR doesn't require it. +- [ ] I updated/added relevant documentation. +- [ ] I listed at least one issue that this PR fixes in the description above. +- [ ] I added new tests to check the change I am making, or this PR is test-exempt. +- [ ] All existing and new tests are passing. + +### Merging: +- Squash merge to dev +- Merge commit to publish-alpha and main + + + +[Documentation]: https://www.100ms.live/docs \ No newline at end of file diff --git a/.github/workflows/alpha-release.yml b/.github/workflows/alpha-release.yml new file mode 100644 index 0000000000..a1388d3b95 --- /dev/null +++ b/.github/workflows/alpha-release.yml @@ -0,0 +1,48 @@ +name: Publish and deploy alpha versions + +on: + push: + branches: + - publish-alpha + +jobs: + bump_versions: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Configure Git user + run: | + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git config user.name "github-actions[bot]" + + - name: Install packages + run: yarn install --frozen-lockfile + + - name: Update versions + run: | + yarn global add lerna@5 + lerna -v + echo $(lerna version prerelease --no-git-tag-version --exact --yes --no-private) + lerna add @100mslive/hms-video-store --peer --scope=@100mslive/hms-virtual-background --exact + lerna add @100mslive/roomkit-react --scope=prebuilt-react-integration --exact + + - name: Commit and push changes + run: | + git add . + git commit -m "build: update versions for release" + git push origin HEAD:publish-alpha + + + run_publish_packages: + runs-on: ubuntu-latest + needs: bump_versions + steps: + - name: Trigger Publish Packages workflow + uses: aurelien-baudet/workflow-dispatch@v4.0.0 + with: + workflow: publish.yml + token: ${{ secrets.GITHUB_TOKEN }} + inputs: '{ "publishFlag": "alpha" }' \ No newline at end of file diff --git a/.github/workflows/create-release-pr.yml b/.github/workflows/create-release-pr.yml index dc496eaaf8..25b42a8a6f 100644 --- a/.github/workflows/create-release-pr.yml +++ b/.github/workflows/create-release-pr.yml @@ -1,21 +1,25 @@ -name: Create release PR +name: Create Release PR on: workflow_dispatch: inputs: versionBump: description: 'which version to bump eg: prerelease, patch' required: true + type: choice default: 'prerelease' + options: + - prerelease + - patch jobs: create_pr: runs-on: ubuntu-latest steps: - name: Validate branch - if: github.event.inputs.versionBump != 'prerelease' && github.ref != 'refs/heads/main' + if: github.event.inputs.versionBump != 'prerelease' && github.ref != 'refs/heads/dev' run: exit 1 - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 @@ -44,12 +48,12 @@ jobs: echo $STORE_VERSION echo "::set-output name=store_version::$(echo $STORE_VERSION)" - - uses: peter-evans/create-pull-request@v4 + - uses: peter-evans/create-pull-request@v7 with: - commit-message: 'build: update versions for release' - title: 'build: update versions for release' + commit-message: 'ci: update versions for release' + title: 'ci: update versions for release' body: | - @100mslive/hms-video-store - ${{ steps.version.outputs.store_version }} - branch: release + branch: ci/release branch-suffix: short-commit-hash delete-branch: true diff --git a/.github/workflows/cypress.yml b/.github/workflows/cypress.yml index 34ed21abac..0ace350abd 100644 --- a/.github/workflows/cypress.yml +++ b/.github/workflows/cypress.yml @@ -8,8 +8,8 @@ jobs: runs-on: ubuntu-latest if: github.event.pull_request.draft != true steps: - - uses: actions/checkout@v2 - - uses: dorny/paths-filter@v2 + - uses: actions/checkout@v4 + - uses: dorny/paths-filter@v3 id: filter with: filters: | diff --git a/.github/workflows/firstinteraction.yml b/.github/workflows/firstinteraction.yml new file mode 100644 index 0000000000..345e65ee85 --- /dev/null +++ b/.github/workflows/firstinteraction.yml @@ -0,0 +1,34 @@ +name: first-interaction + +on: + workflow_dispatch: {} + issues: + types: [opened] + pull_request: + branches: + - main + - develop + types: [opened] + +jobs: + check_for_first_interaction: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/first-interaction@main + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + issue-message: | + Hello! Thank you for filing an issue. + + Please include relevant logs or detailed description for faster resolutions. + + We really appreciate your contribution! + pr-message: | + Hello! Thank you for your contribution. + + If you are fixing a bug, please reference the issue number in the description. + + If you are implementing a feature request, please check with the maintainers that the feature will be accepted first. + + We really appreciate your contribution! diff --git a/.github/workflows/generate-docs.yml b/.github/workflows/generate-docs.yml index 277c0d7ecd..113382818a 100644 --- a/.github/workflows/generate-docs.yml +++ b/.github/workflows/generate-docs.yml @@ -1,16 +1,20 @@ name: Update API Reference on: workflow_dispatch: + workflow_call: + secrets: + DOCKER_GIT_TOKEN: + required: true jobs: generate_api_reference: runs-on: ubuntu-latest steps: - name: Checkout repo - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Use Node ${{ matrix.node }} - uses: actions/setup-node@v3 + uses: actions/setup-node@v4 with: node-version: ${{ matrix.node }} cache: 'yarn' @@ -30,7 +34,7 @@ jobs: run: yarn docs - name: checkout 100ms-docs - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: token: ${{ secrets.DOCKER_GIT_TOKEN }} repository: 100mslive/100ms-docs @@ -48,9 +52,9 @@ jobs: rm -r v2 mv docs v2 mv react/docs v2/react-hooks - + - name: Create PR - uses: peter-evans/create-pull-request@v3 + uses: peter-evans/create-pull-request@v7 with: path: 100ms-docs token: ${{ secrets.DOCKER_GIT_TOKEN }} diff --git a/.github/workflows/lint-test-build.yml b/.github/workflows/lint-test-build.yml index 7bafe43ed7..92c54ad683 100644 --- a/.github/workflows/lint-test-build.yml +++ b/.github/workflows/lint-test-build.yml @@ -1,14 +1,18 @@ name: Lint, Test and Build -on: [push] +on: + push: + merge_group: + types: [checks_requested] + jobs: build: runs-on: ubuntu-latest steps: - name: Checkout repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup Node - uses: actions/setup-node@v3 + uses: actions/setup-node@v4 with: node-version: 18 cache: 'yarn' diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 853a9f91f0..3935706aeb 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -3,9 +3,13 @@ on: workflow_dispatch: inputs: publishFlag: - description: 'which version to publish eg: alpha, beta, latest, experimental' + description: 'which version to publish eg: alpha, latest' required: true default: 'alpha' + type: choice + options: + - alpha + - latest repository_dispatch: types: [publish-command] @@ -13,8 +17,8 @@ jobs: publish_packages: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions/setup-node@v3 + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 with: node-version: 18 registry-url: https://registry.npmjs.org/ @@ -26,7 +30,7 @@ jobs: - name: Notify slack starting if: github.event.inputs.publishFlag == 'latest' && success() id: slack # IMPORTANT: reference this step ID value in future Slack steps - env: + env: SLACK_BOT_TOKEN: ${{ secrets.SLACK_DEPLOY_BOT_TOKEN }} uses: voxmedia/github-action-slack-notify-build@v1 with: @@ -64,6 +68,9 @@ jobs: NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} PUBLISH_FLAG: ${{ github.event.inputs.publishFlag || github.event.client_payload.slash_command.args.unnamed.arg1 || 'alpha' }} + - name: Delay for 15s + run: sleep 15 + - name: Notify slack success if: github.event.inputs.publishFlag == 'latest' && success() env: @@ -85,3 +92,24 @@ jobs: channel_id: ${{ secrets.SLACK_DEPLOY_PROD_CHANNEL_ID }} status: Failed color: danger + + run_api_reference: + needs: publish_packages + if: github.event.inputs.publishFlag == 'latest' + uses: ./.github/workflows/generate-docs.yml + secrets: + DOCKER_GIT_TOKEN: ${{ secrets.DOCKER_GIT_TOKEN}} + + notify_100ms_links: + runs-on: ubuntu-latest + needs: publish_packages + steps: + - name: Repository Dispatch + if: success() + uses: peter-evans/repository-dispatch@v3 + with: + token: ${{ secrets.PAT }} + repository: 100mslive/100ms-links + event-type: alpha-publish + client-payload: '{"bump": "${{ github.event.inputs.publishFlag }}"}' + diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml new file mode 100644 index 0000000000..8e9e78c8c8 --- /dev/null +++ b/.github/workflows/stale.yml @@ -0,0 +1,20 @@ +name: Close stale issues and PRs +on: + workflow_dispatch: {} + schedule: + - cron: "30 1 * * *" + +jobs: + stale: + runs-on: ubuntu-latest + steps: + - uses: actions/stale@v9 + with: + stale-issue-message: "This issue is stale because it has been open 30 days with no activity. Remove stale label or comment or this will be closed in 5 days." + stale-pr-message: "This PR is stale because it has been open 45 days with no activity. Remove stale label or comment or this will be closed in 10 days." + close-issue-message: "This issue was closed because it has been stalled for 5 days with no activity." + close-pr-message: "This PR was closed because it has been stalled for 10 days with no activity." + days-before-issue-stale: 30 + days-before-pr-stale: 45 + days-before-issue-close: 5 + days-before-pr-close: 10 diff --git a/.github/workflows/sync-alpha-to-main.yml b/.github/workflows/sync-alpha-to-main.yml new file mode 100644 index 0000000000..0557263d8f --- /dev/null +++ b/.github/workflows/sync-alpha-to-main.yml @@ -0,0 +1,22 @@ +name: Update publish-alpha +on: + pull_request: + types: [closed] + branches: + - dev + +jobs: + pull-request: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: pull-request + uses: repo-sync/pull-request@v2 + if: github.event.pull_request.merged == true + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + source_branch: 'dev' + destination_branch: 'publish-alpha' + pr_title: 'Update publish-alpha' + pr_body: ':robot: Automated PR from dev to publish-alpha' + pr_label: 'auto-pr' diff --git a/README.md b/README.md index 1c5bbab8ca..05ee429921 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,63 @@ -## Web sdks +# Web SDKs -This monorepo contains all the packages required to integrate 100ms on web +[![Lint, Test and Build](https://github.com/100mslive/web-sdks/actions/workflows/lint-test-build.yml/badge.svg)](https://github.com/100mslive/web-sdks/actions/workflows/lint-test-build.yml) +[![Activity](https://img.shields.io/github/commit-activity/m/100mslive/web-sdks.svg)](https://www.100ms.live/docs/javascript/v2/release-notes/release-notes) +[![License](https://img.shields.io/npm/l/@100mslive/hms-video-store)](https://www.100ms.live/) +[![Documentation](https://img.shields.io/badge/Read-Documentation-blue)](https://www.100ms.live/docs/javascript/v2/quickstart/javascript-quickstart) +[![Register](https://img.shields.io/badge/Contact-Know%20More-blue)](https://dashboard.100ms.live/register) -## Before doing any code change please take time to go through the [guidelines](./DEVELOPER.MD) line by line. +This monorepo contains all the packages required to integrate 100ms on the web. + +## What is included? + +The packages folder contains all the SDK's of 100ms. Here is a brief overview of them: + +| Directory | Package | Description | Link | +|--|--|--|--| +| `hms-video-store` | `@100mslive/hms-video-store` | This package contains the core SDK and the reactive store parts. | [README](./packages/hms-video-store) | +| `react-sdk` | `@100mslive/react-sdk` | This contains the base React Hooks and some commonly used functionalities as React Hooks. | [README](./packages/react-sdk) | +| `roomkit-react` | `@100mslive/roomkit-react`| This contains the React components used in the Prebuilt and the Prebuilt component itself. | [README](./packages/roomkit-react) | +| `roomkit-web` | `@100mslive/roomkit-web` | This is a web component port of the `HMSPrebuilt` component from the `roomkit-react`. If you are not using React, this can be used as a web component. | [README](./packages/roomkit-web)| +| `hls-player` | `@100mslive/hls-player` | This is a HLS player offered by 100ms that can be used to play live video streams. | [README](./packages/hls-player) | +| `hms-whiteboard` | `@100mslive/hms-whiteboard` | This contains APIs for integrating Whiteboard collaboration into your conferencing sessions. | [README](./packages/hms-whiteboard) | +| `hms-virtual-background` | `@100mslive/hms-virtual-background` | This contains the Virtual Background plugin provided by 100ms. | [README](./packages/hms-virtual-background) | +| `react-icons` | `@100mslive/react-icons` | This contains all the icons used in the 100ms prebuilt. | [README](./packages/react-icons) | + +For full documentation, visit [100ms.live/docs](https://www.100ms.live/docs) + +## How to integrate? + +The 100ms SDK gives you everything you need to build scalable, high-quality live video and audio experiences. + +**There are two ways you can add 100ms to your apps:** + +1. ## Custom UI + - 100ms SDKs are powerful and highly extensible to build and support all custom experiences and UI. + - **Related packages include:** `@100mslive/react-sdk`, `@100mslive/hms-video-store` and `@100mslive/react-icons`. + - Get started with integrating the SDK using the [How to Guide](https://www.100ms.live/docs/javascript/v2/how-to-guides/install-the-sdk/integration). + +> Navigate to `react-sdk` for the base React Hooks and some commonly used functionalities by clicking [here](./packages/react-sdk). + +2. ## 100ms Prebuilt + - 100ms Prebuilt is a high-level abstraction with no-code customization that enables you to embed video conferencing and/or live streaming UI—with a few lines of code. + - **Related packages include:** `roomkit-react` and `roomkit-web`. + - Get started with 100ms Prebuilt using the [Prebuilt Quickstart for Web](https://www.100ms.live/docs/javascript/v2/quickstart/prebuilt-quickstart). + +> Navigate to `roomkit-react` for the React components used in Prebuilt and the Prebuilt component itself by clicking [here](./packages/roomkit-react). + +![Banner](prebuilt-banner.png) + +### 100ms Prebuilt Cross Platform Support + +| Client | Repository | Docs | Example | +| ------------ | -------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | +| Web | [web-sdks](https://github.com/100mslive/web-sdks/tree/main/packages/roomkit-react) | [Link](https://www.100ms.live/docs/javascript/v2/quickstart/prebuilt-quickstart) | [prebuilt-react-integration](https://github.com/100mslive/web-sdks/tree/main/examples/prebuilt-react-integration) | +| Android | [100ms-android](https://github.com/100mslive/100ms-android/tree/release-v2/room-kit) | [Link](https://www.100ms.live/docs/android/v2/quickstart/prebuilt-android) | [AndroidPrebuiltDemo](https://github.com/100mslive/AndroidPrebuiltDemo) | +| iOS | [100ms-roomkit-ios](https://github.com/100mslive/100ms-roomkit-ios) | [Link](https://www.100ms.live/docs/ios/v2/quickstart/prebuilt) | [100ms-roomkit-example](https://github.com/100mslive/100ms-roomkit-example) | +| Flutter | [100ms-flutter](https://github.com/100mslive/100ms-flutter/tree/main/packages/hms_room_kit) | [Link](https://www.100ms.live/docs/flutter/v2/quickstart/prebuilt) | [hms_room_kit/example](https://github.com/100mslive/100ms-flutter/tree/main/packages/hms_room_kit/example) | +| React Native | [100ms-react-native](https://github.com/100mslive/100ms-react-native/tree/main/packages/react-native-room-kit) | [Link](https://www.100ms.live/docs/react-native/v2/quickstart/prebuilt) | [react-native-room-kit/example](https://github.com/100mslive/100ms-react-native/tree/main/packages/react-native-room-kit/example) | + +## Setup ### Local Setup @@ -10,8 +65,7 @@ This monorepo contains all the packages required to integrate 100ms on web if you are using a different version in other projects, use [nvm](https://github.com/nvm-sh/nvm?tab=readme-ov-file#installing-and-updating) to manage node versions. - -``` +```bash git clone https://github.com/100mslive/web-sdks.git cd web-sdks yarn install @@ -20,7 +74,7 @@ yarn build ### Running Sample Prebuilt -``` +```bash cd examples/prebuilt-react-integration yarn dev ``` @@ -29,43 +83,64 @@ yarn dev Once you run `yarn dev`, the localhost link with the port will be generated automatically. Just get the roomCode from [100ms dashboard](https://dashboard.100ms.live) and append at the end +### Testing Changes Locally -### Testing changes locally Run `yarn start` by navigating to the package you are making changes to, the changes should reflect in the above sample app. For example, if you are making changes in roomkit-react(prebuilt), run `yarn start` in that package. The sample app should auto reload. -> Note: Make sure `yarn build` is run atleast once before using `yarn start` +> Note: Make sure `yarn build` is run atleast once before using `yarn start`. -### Understanding the packages: -The packages folder contains all the SDK's of 100ms. Here is a brief overview of them. +### Deploying Your Changes -`hms-video-store` -is the source of `@100mslive/hms-video-store`. -This package contains the core SDK and the reactive store parts. -For more details refer [here](https://github.com/100mslive/web-sdks/blob/main/packages/hms-video-store/README.md). +Once you have forked the repository and tested your changes on the local build, you can follow the steps below to deploy to Vercel as an example: -`react-icons` -is the source of `@100mslive/react-icons`. -This contains all the icons used in the 100ms prebuilt. -For more details refer [here](https://github.com/100mslive/web-sdks/blob/main/packages/react-icons/README.md). +- Import the fork repository +- Set `examples/prebuilt-react-integration` as the root directory +- Use the Create React App preset and update the build and install commands `Project Settings` to use the root level scripts + - install: `cd ../../ && yarn install` + - build: `cd ../../ && yarn build` -`react-sdk` -is the source of `@100mslive/react-sdk`. -This contains the base React Hooks and some commonly used functionalities as React Hooks. -For more details refer [here](https://github.com/100mslive/web-sdks/blob/main/packages/react-sdk/README.md). +For reference: +![Project Settings](./project-settings.png) -`roomkit-react` -is the source of `@100mslive/roomkit-react`. -This contains the React components used in the prebuilt and the Prebuilt component itself. -For more details refer [here](https://github.com/100mslive/web-sdks/blob/main/packages/roomkit-react/README.md). -`roomkit-web` -is the source of `@100mslive/roomkit-web`. -This is a web component port of the `HMSPrebuilt` component from the `roomkit-react`. If you are not using React, -this can be used as a web component. -For more details refer [here](https://github.com/100mslive/web-sdks/blob/main/packages/roomkit-web/README.md). +Once the app has been deployed, you can append the room code at the end of the deployment URL to preview your changes + +### Maintaining A Forked Version + +The following command will build the roomkit-react package and generate a .tgz file: + +```bash +yarn && yarn build; +cd packages/roomkit-react; +yarn pack +``` + +Push your changes and the .tgz file to the forked repository and in the package.json of the app you are building, use the following link for the roomkit-react package: + +``` +"@100mslive/roomkit-react":"https://github.com//web-sdks/raw/main/packages/roomkit-react/.tgz", +``` + +Re-install the dependencies after updating the package.json and build using the following command: + +``` +yarn && yarn build +``` + +You can now import the HMSPrebuilt component in the same way as before: + +``` +import { HMSPrebuilt } from '@100mslive/roomkit-react'; +``` + +## Contributing +We welcome external contributors or anyone excited to help improve 100ms SDKs. If you'd like to get involved, check out our [contribution guide](./DEVELOPER.MD), and get started exploring the codebase. +## Community & Support +- [GitHub Issues](https://github.com/100mslive/web-sdks/issues) - Submit any bugs or errors you encounter using the Web SDKs. +- [Contact](https://www.100ms.live/contact) - Reach out to 100ms team to get pricing information, understand how we can help you go live, or to learn more about the platform. diff --git a/examples/prebuilt-react-integration/README.md b/examples/prebuilt-react-integration/README.md index 375059f0b5..244c6693f5 100644 --- a/examples/prebuilt-react-integration/README.md +++ b/examples/prebuilt-react-integration/README.md @@ -1,5 +1,8 @@ -### Running locally +![Banner](https://github.com/100mslive/web-sdks/blob/06c65259912db6ccd8617f2ecb6fef51429251ec/prebuilt-banner.png) -run `yarn build` at the root level of the repo. +# 100ms Prebuilt - React Example -Then navigate to this folder and run `yarn dev` \ No newline at end of file +### How to run locally? + +- Run `yarn && yarn build` at the root level of the repo (./web-sdks) +- Then, navigate to this folder and run `yarn dev` diff --git a/examples/prebuilt-react-integration/index.html b/examples/prebuilt-react-integration/index.html index 895cbcd8ce..aee5d151bb 100644 --- a/examples/prebuilt-react-integration/index.html +++ b/examples/prebuilt-react-integration/index.html @@ -1,4 +1,4 @@ - + diff --git a/examples/prebuilt-react-integration/package.json b/examples/prebuilt-react-integration/package.json index 1e1da55eb5..643cb76447 100644 --- a/examples/prebuilt-react-integration/package.json +++ b/examples/prebuilt-react-integration/package.json @@ -10,7 +10,7 @@ "preview": "vite preview" }, "dependencies": { - "@100mslive/roomkit-react": "0.2.6", + "@100mslive/roomkit-react": "0.3.25", "react": "^18.2.0", "react-dom": "^18.2.0" }, diff --git a/examples/prebuilt-react-integration/src/App.jsx b/examples/prebuilt-react-integration/src/App.jsx index e49ddcb9b1..f46d5dce77 100644 --- a/examples/prebuilt-react-integration/src/App.jsx +++ b/examples/prebuilt-react-integration/src/App.jsx @@ -1,8 +1,13 @@ -import { HMSPrebuilt } from '@100mslive/roomkit-react'; +import { HMSPrebuilt, Diagnostics } from '@100mslive/roomkit-react'; import { getRoomCodeFromUrl } from './utils'; export default function App() { const roomCode = getRoomCodeFromUrl(); + const isDiagnostics = location.pathname.startsWith('/diagnostics'); + + if (isDiagnostics) { + return ; + } return ; } diff --git a/examples/prebuilt-react-integration/src/main.css b/examples/prebuilt-react-integration/src/main.css index a38f4942b1..83b726cacd 100644 --- a/examples/prebuilt-react-integration/src/main.css +++ b/examples/prebuilt-react-integration/src/main.css @@ -1,3 +1,5 @@ +@import url('@100mslive/roomkit-react/index.css'); + html, body, #root { @@ -16,4 +18,8 @@ body { left: 0; width: 100%; margin: 0; -} \ No newline at end of file +} + +.tl-container { + border-radius: 0.75rem; +} diff --git a/examples/prebuilt-react-integration/vite.config.js b/examples/prebuilt-react-integration/vite.config.js index 8c3199c2b7..09395b4901 100644 --- a/examples/prebuilt-react-integration/vite.config.js +++ b/examples/prebuilt-react-integration/vite.config.js @@ -1,10 +1,27 @@ import { defineConfig } from 'vite'; import react from '@vitejs/plugin-react-swc'; +import { basename } from 'path'; +import fs from 'fs'; + +// Context: https://github.com/tensorflow/tfjs/issues/7165 +function mediapipe_workaround() { + return { + name: 'mediapipe_workaround', + load(id) { + if (basename(id) === 'selfie_segmentation.js') { + let code = fs.readFileSync(id, 'utf-8'); + code += 'exports.SelfieSegmentation = SelfieSegmentation;'; + return { code }; + } + return null; + }, + }; +} // https://vitejs.dev/config/ export default defineConfig({ - plugins: [react()], + plugins: [react(), mediapipe_workaround()], define: { 'process.env': {}, - } + }, }); diff --git a/lerna.json b/lerna.json index af5e816a1d..1075160d5c 100644 --- a/lerna.json +++ b/lerna.json @@ -1,8 +1,7 @@ { "packages": [ "packages/*", - "apps/*", - "playwright/*" + "examples/*" ], "version": "independent", "npmClient": "yarn", @@ -13,6 +12,7 @@ "message": "build: update version" }, "publish": { + "access": "public", "registry": "https://registry.npmjs.org/" } }, diff --git a/package.json b/package.json index 9958ebf520..652b546ff7 100644 --- a/package.json +++ b/package.json @@ -3,8 +3,8 @@ "private": true, "devDependencies": { "@babel/core": "^7.8.0", - "@commitlint/cli": "^15.0.0", - "@commitlint/config-conventional": "^15.0.0", + "@commitlint/cli": "^19.6.0", + "@commitlint/config-conventional": "^19.6.0", "@size-limit/file": "^5.0.3", "@types/jest": "^27.0.3", "@types/node": "^16.11.17", @@ -58,7 +58,8 @@ "ybys": "yarn && yarn build --no-private && yarn storybook" }, "resolutions": { - "loader-utils": "^2.0.4" + "loader-utils": "^2.0.4", + "axios": "^1.7.4" }, "workspaces": [ "packages/*", diff --git a/packages/hls-player/README.md b/packages/hls-player/README.md index 0dda1052f8..0f8e8f1dd5 100644 --- a/packages/hls-player/README.md +++ b/packages/hls-player/README.md @@ -1,17 +1,348 @@ -`@100mslive/hls-player` is currently a wrapper on hls.js with easy to use interface and few add-ons for [100ms's interactive live streaming feature](https://www.100ms.live/docs/javascript/v2/how--to-guides/record-and-live-stream/hls/hls). +# 100ms HLS Player -Sample usage: +[![Lint, Test and Build](https://github.com/100mslive/web-sdks/actions/workflows/lint-test-build.yml/badge.svg)](https://github.com/100mslive/web-sdks/actions/workflows/lint-test-build.yml) +[![Bundle Size](https://badgen.net/bundlephobia/minzip/@100mslive/hls-player)](https://bundlephobia.com/result?p=@100mslive/hls-player) +[![License](https://img.shields.io/npm/l/@100mslive/hls-player)](https://www.100ms.live/) +![Tree shaking](https://badgen.net/bundlephobia/tree-shaking/@100mslive/hls-player) +The `HMSHLSPlayer` is an HLS player offered by 100ms that can be used to play HLS streams. The player takes a URL and video element to play the stream. + +## How to integrate HLS Player SDK + +You can use Node package manager or yarn to add HMSHLSPlayer sdk to your project. +Use [@100mslive/hls-player](https://www.npmjs.com/package/@100mslive/hls-player) as the package source. + +```bash +npm i @100mslive/hls-player +``` + +## HMSHLSPlayer methods + +Below shows all the methods exposed from player + +```javascript +interface IHMSHLSPlayer { + /** + * @returns get html video element + */ + getVideoElement(): HTMLVideoElement; + + /** + * set video volumne + * @param { volume } - in range [0,100] + */ + setVolume(volume: number): void; + /** + * + * @returns returns HMSHLSLayer which represents current + * quality. + */ + getLayer(): HMSHLSLayer | null; + /** + * + * @param { HMSHLSLayer } layer - layer we want to set the stream to. + * set { height: auto } to set the layer to auto + */ + setLayer(layer: HMSHLSLayer): void; + /** + * move the video to Live + */ + seekToLivePosition(): Promise; + /** + * play stream + * call this when autoplay error is received + */ + play(): Promise; + /** + * pause stream + */ + pause(): void; + + /** + * It will update the video element current time + * @param seekValue Pass currentTime in second + */ + seekTo(seekValue: number): void; +} +``` + +### How to use Player's HLS Stream + +You create an instance of HMSHLSPlayer like below: + +```javascript +import { HMSHLSPlayer } from '@100mslive/hls-player'; + +// hls url should be provided which player will run. +// second parameter is optional, if you had video element then pass to player else we will create one. +const hlsPlayer = new HMSHLSPlayer(hlsURL, videoEl) + +// if video element is not present, we will create a new video element which can be attached to your ui. +const videoEl = hlsPlayer.getVideoElement(); +``` + +### How to pause and resume the playback + +You call play/pause on the hlsPlayer instance like below: + +```javascript +// return Promise +hmsPlayer.play() + +hmsPlayer.pause() +``` + +### How to seek forward or backward + +You use `seekTo` methods on the hlsPlayer to seek to any position in video, below is given example: + +```javascript +// seekValue Pass currentTime in second +hmsPlayer.seekTo(5) +``` + +### How to seek to live position + +You use `seekToLivePosition` methods on the hlsPlayer instance to go to the live poition like below: + +```javascript +hmsPlayer.seekToLivePosition() +``` + +### How to change volume of HLS playback + +Use volume property to change the volume of HLS player. The volume level is between 0-100. Volume is set to 100 by default. + +```javascript +hlsPlayer.setVolume(50); ``` -import { - HLSPlaybackState, -} from "@100mslive/hls-player"; -// hlsUrl is the url in which the hls stream is ongoing -// videoElement is the video element where you want to play the stream -const player = new HMSHLSPlayer(hlsUrl, videoElement); -player.play() +### Set video quality level to hls player +```javascript +/** +* +* @returns returns HMSHLSLayer which represents current +* quality. +*/ +hlsPlayer.getLayer(): HMSHLSLayer | null; +/** +* +* @param { HMSHLSLayer } layer - layer we want to set the stream to. +* set { height: auto } to set the layer to auto +*/ +hlsPlayer.setLayer(layer: HMSHLSLayer): void; + +// quality interface +interface HMSHLSLayer { + readonly bitrate: number; + readonly height?: number; + readonly id?: number; + readonly width?: number; + url: string; + resolution?: string; +} ``` -More details to be added soon. +## Events exposed from HMSHLSPlayer + +We are exposing events from our hls player. + +```javascript +enum HMSHLSPlayerEvents { + TIMED_METADATA_LOADED = 'timed-metadata', + SEEK_POS_BEHIND_LIVE_EDGE = 'seek-pos-behind-live-edge', + + CURRENT_TIME = 'current-time', + AUTOPLAY_BLOCKED = 'autoplay-blocked', + + MANIFEST_LOADED = 'manifest-loaded', + LAYER_UPDATED = 'layer-updated', + + ERROR = 'error', + PLAYBACK_STATE = 'playback-state', + STATS = 'stats', +} +``` + +### Playback state + +```javascript +enum HLSPlaybackState { + playing, + paused, +} +interface HMSHLSPlaybackState { + state: HLSPlaybackState; +} +hlsPlayer.on(HMSHLSPlayerEvents.PLAYBACK_STATE, (event: HMSHLSPlayerEvents, data: HMSHLSPlaybackState): void => {}); +``` + +### HLS Stats + +```javascript +interface HlsPlayerStats { + /** Estimated bandwidth in bits/sec. Could be used to show connection speed. */ + bandwidthEstimate?: number; + /** The bitrate of the current level that is playing. Given in bits/sec */ + bitrate?: number; + /** the amount of video available in forward buffer. Given in ms */ + bufferedDuration?: number; + /** how far is the current playback from live edge.*/ + distanceFromLive?: number; + /** total Frames dropped since started watching the stream. */ + droppedFrames?: number; + /** the m3u8 url of the current level that is being played */ + url?: string; + /** the resolution of the level of the video that is being played */ + videoSize?: { + height: number; + width: number; + }; +} + +hlsPlayer.on(HMSHLSPlayerEvents.STATS, (event: HMSHLSPlayerEvents, data: HlsPlayerStats): void => {}); +``` + +### Manifest loaded data + +Hls player will provide a manifest which will provide a data like different quality layer once url is loaded. + +```javascript +interface HMSHLSManifestLoaded { + layers: HMSHLSLayer[]; +} +hlsPlayer.on(HMSHLSPlayerEvents.MANIFEST_LOADED, (event: HMSHLSPlayerEvents, data: HMSHLSManifestLoaded): void => {}); +``` + +### Quality changed data + +```javascript +interface HMSHLSLayerUpdated { + layer: HMSHLSLayer; +} +hlsPlayer.on(HMSHLSPlayerEvents.LAYER_UPDATED, (event: HMSHLSPlayerEvents, data: HMSHLSLayerUpdated): void => {}); +``` + +### Live Event + +Player will let you know if player is plaaying video live or not + +```javascript +interface HMSHLSStreamLive { + isLive: boolean; +} +hlsPlayer.on(HMSHLSPlayerEvents.SEEK_POS_BEHIND_LIVE_EDGE, (event: HMSHLSPlayerEvents, data: HMSHLSStreamLive): void => {}); +``` + +### HLS timed-metadata + +HLS player will parse and send the timed-metadata. + +```javascript +interface HMSHLSCue { + id?: string; + payload: string; + duration: number; + startDate: Date; + endDate?: Date; +} +hlsPlayer.on(HMSHLSPlayerEvents.TIMED_METADATA_LOADED, (event: HMSHLSPlayerEvents, data: HMSHLSCue): void => {}); +``` + +### Error handling + +```javascript +interface HMSHLSException { + name: string, + message: string, + description: string, + isTerminal: boolean, // decide if player error will automatically restart(if false) +} +hlsPlayer.on(HMSHLSPlayerEvents.ERROR, (event: HMSHLSPlayerEvents, data: HMSHLSException): void => {}); +hlsPlayer.on(HMSHLSPlayerEvents.AUTOPLAY_BLOCKED, (event: HMSHLSPlayerEvents, data: HMSHLSException): void => {}); +``` + +### Video current time + +```javascript +hlsPlayer.on(HMSHLSPlayerEvents.CURRENT_TIME, (event: HMSHLSPlayerEvents, data: number): void => {}); +``` + +### Example for events usage + +Below are the simple example of how to use hls player's event + +```javascript +const isPlaying = false; +const playbackEventHandler = data => isPlaying = data.state === HLSPlaybackState.paused; +hlsPlayer.on(HMSHLSPlayerEvents.PLAYBACK_STATE, playbackEventHandler); +``` + +## HLS Player example + +Below is a simple example in which hls-player will be used in your app. + +```javascript +// Vanilla JavaScript Example +import { HLSPlaybackState, HMSHLSPlayer, HMSHLSPlayerEvents } from "@100mslive/hls-player"; + +const videoEl; // reference for video element +const hlsUrl; // reference to hls url + +// variable to handle ui and take some actions +let isLive = true, isPaused = false, isAutoBlockedPaused = false; + +const handleError = data => console.error("[HLSView] error in hls", data); +const handleNoLongerLive = ({ isLive }) => isLive = isLive; + +const playbackEventHandler = data => isPaused = (data.state === HLSPlaybackState.paused); + +const handleAutoplayBlock = data => isAutoBlockedPaused = !!data; + +const hlsPlayer = new HMSHLSPlayer(hlsUrl, videoEl); + +hlsPlayer.on(HMSHLSPlayerEvents.SEEK_POS_BEHIND_LIVE_EDGE, handleNoLongerLive); +hlsPlayer.on(HMSHLSPlayerEvents.ERROR, handleError); +hlsPlayer.on(HMSHLSPlayerEvents.PLAYBACK_STATE, playbackEventHandler); +hlsPlayer.on(HMSHLSPlayerEvents.AUTOPLAY_BLOCKED, handleAutoplayBlock); +``` + +```jsx +// React Example +import { HLSPlaybackState, HMSHLSPlayer, HMSHLSPlayerEvents } from "@100mslive/hls-player"; +import { useEffect, useState } from 'react'; + +const videoEl; // reference for video element +const hlsUrl; // reference to hls url + +// variable to handle ui and take some actions +const [isVideoLive, setIsVideoLive] = useState(true); +const [isHlsAutoplayBlocked, setIsHlsAutoplayBlocked] = useState(false); +const [isPaused, setIsPaused] = useState(false); + +useEffect(() => { + const handleError = data => console.error("[HLSView] error in hls", data); + const handleNoLongerLive = ({ isLive }) => { + setIsVideoLive(isLive); + }; + + const playbackEventHandler = data => + setIsPaused(data.state === HLSPlaybackState.paused); + + const handleAutoplayBlock = data => setIsHlsAutoplayBlocked(!!data); + const hlsPlayer = new HMSHLSPlayer(hlsUrl, videoEl); + + hlsPlayer.on(HMSHLSPlayerEvents.SEEK_POS_BEHIND_LIVE_EDGE, handleNoLongerLive); + hlsPlayer.on(HMSHLSPlayerEvents.ERROR, handleError); + hlsPlayer.on(HMSHLSPlayerEvents.PLAYBACK_STATE, playbackEventHandler); + hlsPlayer.on(HMSHLSPlayerEvents.AUTOPLAY_BLOCKED, handleAutoplayBlock); + return () => { + hlsPlayer.off(HMSHLSPlayerEvents.SEEK_POS_BEHIND_LIVE_EDGE, handleNoLongerLive); + hlsPlayer.off(HMSHLSPlayerEvents.ERROR, handleError); + hlsPlayer.off(HMSHLSPlayerEvents.PLAYBACK_STATE, playbackEventHandler); + hlsPlayer.off(HMSHLSPlayerEvents.AUTOPLAY_BLOCKED, handleAutoplayBlock); + } +}, []); + +``` diff --git a/packages/hls-player/package.json b/packages/hls-player/package.json index af8ed212eb..b0e2fff8e9 100644 --- a/packages/hls-player/package.json +++ b/packages/hls-player/package.json @@ -1,6 +1,6 @@ { "name": "@100mslive/hls-player", - "version": "0.2.6", + "version": "0.3.25", "description": "HLS client library which uses HTML5 Video element and Media Source Extension for playback", "main": "dist/index.cjs.js", "module": "dist/index.js", @@ -36,8 +36,16 @@ "author": "100ms", "license": "MIT", "dependencies": { - "@100mslive/hls-stats": "0.3.6", + "@100mslive/hls-stats": "0.4.25", "eventemitter2": "^6.4.9", "hls.js": "1.4.12" - } + }, + "keywords": [ + "hls", + "video", + "player", + "webrtc", + "conferencing", + "100ms" + ] } diff --git a/packages/hls-player/src/controllers/HMSHLSPlayer.ts b/packages/hls-player/src/controllers/HMSHLSPlayer.ts index fea74ec1f9..18a24f4bdb 100644 --- a/packages/hls-player/src/controllers/HMSHLSPlayer.ts +++ b/packages/hls-player/src/controllers/HMSHLSPlayer.ts @@ -236,7 +236,7 @@ export class HMSHLSPlayer implements IHMSHLSPlayer, IHMSHLSPlayerEventEmitter { }); }; private volumeEventHandler = () => { - this._volume = this._videoEl.volume; + this._volume = Math.round(this._videoEl.volume * 100); }; private reConnectToStream = () => { @@ -277,7 +277,7 @@ export class HMSHLSPlayer implements IHMSHLSPlayer, IHMSHLSPlayerEventEmitter { this.emitEvent(HMSHLSPlayerEvents.ERROR, error); break; } - // Below one are network related errors + // Below ones are network related errors case Hls.ErrorDetails.MANIFEST_LOAD_ERROR: { const error = HMSHLSErrorFactory.HLSNetworkError.manifestLoadError(detail); this.emitEvent(HMSHLSPlayerEvents.ERROR, error); diff --git a/packages/hls-stats/README.md b/packages/hls-stats/README.md index d422d6ecd0..1af6ec5ed8 100644 --- a/packages/hls-stats/README.md +++ b/packages/hls-stats/README.md @@ -1,5 +1,11 @@ # @100mslive/hms-stats +[![Lint, Test and Build](https://github.com/100mslive/web-sdks/actions/workflows/lint-test-build.yml/badge.svg)](https://github.com/100mslive/web-sdks/actions/workflows/lint-test-build.yml) +[![Bundle Size](https://badgen.net/bundlephobia/minzip/@100mslive/hls-stats)](https://bundlephobia.com/result?p=@100mslive/hls-stats) +[![License](https://img.shields.io/npm/l/@100mslive/hls-stats)](https://www.100ms.live/) +![Tree shaking](https://badgen.net/bundlephobia/tree-shaking/@100mslive/hls-stats) + + A simple library for HLS stats for Hls.js. ## Installation diff --git a/packages/hls-stats/package.json b/packages/hls-stats/package.json index 11a33ecba2..6b7f077190 100644 --- a/packages/hls-stats/package.json +++ b/packages/hls-stats/package.json @@ -1,6 +1,6 @@ { "name": "@100mslive/hls-stats", - "version": "0.3.6", + "version": "0.4.25", "description": "A simple library that provides stats for your hls stream", "main": "dist/index.cjs.js", "module": "dist/index.js", diff --git a/packages/hls-stats/src/adapters/BaseAdapter.ts b/packages/hls-stats/src/adapters/BaseAdapter.ts index 2064a28be5..60a7dcf062 100644 --- a/packages/hls-stats/src/adapters/BaseAdapter.ts +++ b/packages/hls-stats/src/adapters/BaseAdapter.ts @@ -3,10 +3,11 @@ import { HlsPlayerStats } from '../interfaces'; export abstract class BaseAdapter { hlsInstance: Hls; videoEl: HTMLVideoElement; - hlsStatsState: HlsPlayerStats = {}; + hlsStatsState: HlsPlayerStats; constructor(hlsInstance: Hls, videoEl: HTMLVideoElement) { this.hlsInstance = hlsInstance; this.videoEl = videoEl; + this.hlsStatsState = {}; } abstract startGatheringStats(): void; abstract finishGatheringStats(): void; diff --git a/packages/hms-video-store/README.md b/packages/hms-video-store/README.md index b3f3f333f1..1d2f8b5f8e 100644 --- a/packages/hms-video-store/README.md +++ b/packages/hms-video-store/README.md @@ -1,8 +1,9 @@ # 100ms Reactive Store [![NPM](https://badgen.net/npm/v/@100mslive/hms-video-store?color=green)](https://www.npmjs.com/package/@100mslive/hms-video-store) -![Test](https://github.com/100mslive/hms-video-store/actions/workflows/main.yaml/badge.svg) +[![Lint, Test and Build](https://github.com/100mslive/web-sdks/actions/workflows/lint-test-build.yml/badge.svg)](https://github.com/100mslive/web-sdks/actions/workflows/lint-test-build.yml) [![Bundle Size](https://badgen.net/bundlephobia/minzip/@100mslive/hms-video-store)](https://bundlephobia.com/result?p=@100mslive/hms-video-store) +[![License](https://img.shields.io/npm/l/@100mslive/hms-video-store)](https://www.100ms.live/) ![Tree shaking](https://badgen.net/bundlephobia/tree-shaking/@100mslive/hms-video-store) ![Architecture](images/architecture.png) @@ -34,10 +35,10 @@ you want to do - passed in selector such that whenever the portion changes, the passed in callback is notified. 2. Actions - The actions interface for dispatching actions which in turn may reach out to server and update the store. Check the interface with detailed doc - [here](src/core/IHMSActions.ts). + [here](./src/IHMSActions.ts). We also provide optimized and efficient selectors for most common use cases. These are -available in [this folder](src/core/selectors). +available in [this folder](./src/selectors). Important Note: The data received from either getState or Subscribe is immutable, the object received is frozen, and it is not allowed to mutate it. You'll get an error diff --git a/packages/hms-video-store/jest.config.js b/packages/hms-video-store/jest.config.js index c2abcb0929..5c19a40542 100644 --- a/packages/hms-video-store/jest.config.js +++ b/packages/hms-video-store/jest.config.js @@ -1,4 +1,4 @@ -module.exports = { +const config = { transform: { '.(ts|tsx)$': '../../node_modules/ts-jest/dist/index.js', '.(js|jsx)$': '../../node_modules/babel-jest/build/index.js', @@ -9,3 +9,5 @@ module.exports = { setupFiles: ['jest-canvas-mock', 'jsdom-worker'], testEnvironment: 'jsdom', }; + +module.exports = config; diff --git a/packages/hms-video-store/package.json b/packages/hms-video-store/package.json index 86dbeb3f86..8499782ead 100644 --- a/packages/hms-video-store/package.json +++ b/packages/hms-video-store/package.json @@ -1,5 +1,5 @@ { - "version": "0.11.6", + "version": "0.12.25", "license": "MIT", "repository": { "type": "git", diff --git a/packages/hms-video-store/src/IHMSActions.ts b/packages/hms-video-store/src/IHMSActions.ts index e7f2bc488a..bcfe6dbdbc 100644 --- a/packages/hms-video-store/src/IHMSActions.ts +++ b/packages/hms-video-store/src/IHMSActions.ts @@ -1,3 +1,7 @@ +import { HMSDiagnosticsInterface } from './diagnostics/interfaces'; +import { TranscriptionConfig } from './interfaces/transcription-config'; +import { FindPeerByNameRequestParams } from './signal/interfaces'; +import { HMSSessionFeedback } from './end-call-feedback'; import { HLSConfig, HLSTimedMetadata, @@ -21,9 +25,11 @@ import { TokenRequestOptions, } from './internal'; import { + DebugInfo, HMSChangeMultiTrackStateParams, HMSGenericTypes, HMSMessageID, + HMSPeer, HMSPeerID, HMSPeerListIterator, HMSPeerListIteratorOptions, @@ -34,6 +40,7 @@ import { IHMSSessionStoreActions, } from './schema'; import { HMSRoleChangeRequest } from './selectors'; +import { HMSStats } from './webrtc-stats'; /** * The below interface defines our SDK API Surface for taking room related actions. @@ -164,12 +171,14 @@ export interface IHMSActions): Promise; + /** * Change settings of the local peer's video track * @param settings HMSVideoTrackSettings * `({ width, height, codec, maxFramerate, maxBitrate, deviceId, advanced, facingMode })` */ setVideoSettings(settings: Partial): Promise; + /** * Toggle the camera between front and back if the both the camera's exist */ @@ -225,16 +234,16 @@ export interface IHMSActions; /** * Remove video plugins to the local peer video stream. Eg. Virtual Background, Face Filters etc. * Video plugins can be added/removed at any time after the video track is available. - * @param plugin HMSMediaStreamPlugin * @see HMSMediaStreamPlugin + * @param plugins */ removePluginsFromVideoStream(plugins: HMSMediaStreamPlugin[]): Promise; @@ -334,6 +343,13 @@ export interface IHMSActions Promise; + /** + * After leave send feedback to backend for call quality purpose. + * @param feedback + * @param eventEndpoint + */ + submitSessionFeedback(feedback: HMSSessionFeedback, eventEndpoint?: string): Promise; + /** * If you have **removeOthers** permission, you can remove a peer from the room. * @param peerID peerID of the peer to be removed from the room @@ -370,6 +386,18 @@ export interface IHMSActions; + /** + * If you want to start transcriptions(Closed Caption). + * @param params.mode This is the mode which represent the type of transcription. Currently we have Caption mode only + */ + startTranscription(params: TranscriptionConfig): Promise; + + /** + * If you want to stop transcriptions(Closed Caption). + * @param params.mode This is the mode which represent the type of transcription you want to stop. Currently we have Caption mode only + */ + stopTranscription(params: TranscriptionConfig): Promise; + /** * @alpha * Used to define date range metadata in a media playlist. @@ -546,9 +574,24 @@ export interface IHMSActions; lowerRemotePeerHand(peerId: string): Promise; getPeerListIterator(options?: HMSPeerListIteratorOptions): HMSPeerListIterator; + getPeer(peerId: string): Promise; + findPeerByName(options: FindPeerByNameRequestParams): Promise<{ offset: number; eof?: boolean; peers: HMSPeer[] }>; /** * Method to override the default settings for playlist tracks * @param {HMSPlaylistSettings} settings */ setPlaylistSettings(settings: HMSPlaylistSettings): void; + + initDiagnostics(): HMSDiagnosticsInterface; + /** + * @internal + * Method to get enabled flags and endpoints. Should only be called after joining. + */ + getDebugInfo(): DebugInfo | undefined; + + /** + * @internal + * Method to check if received bitrate is 0 for all remote peers or whether the room has whiteboard/quiz running. To be used by beam. + */ + hasActiveElements(hmsStats: HMSStats): boolean; } diff --git a/packages/hms-video-store/src/analytics/AnalyticsEventFactory.ts b/packages/hms-video-store/src/analytics/AnalyticsEventFactory.ts index 1c897ab980..4197182456 100644 --- a/packages/hms-video-store/src/analytics/AnalyticsEventFactory.ts +++ b/packages/hms-video-store/src/analytics/AnalyticsEventFactory.ts @@ -135,6 +135,16 @@ export default class AnalyticsEventFactory { }); } + static audioRecovered(message: string) { + return new AnalyticsEvent({ + name: 'audioRecovered', + level: AnalyticsEventLevel.VERBOSE, + properties: { + message, + }, + }); + } + static deviceChange({ isUserSelection, selection, @@ -235,6 +245,50 @@ export default class AnalyticsEventFactory { }); } + static getKrispUsage(duration: number) { + return new AnalyticsEvent({ + name: 'krisp.usage', + level: AnalyticsEventLevel.INFO, + properties: { duration }, + }); + } + + static krispStart() { + return new AnalyticsEvent({ + name: 'krisp.start', + level: AnalyticsEventLevel.INFO, + }); + } + + static krispStop() { + return new AnalyticsEvent({ + name: 'krisp.stop', + level: AnalyticsEventLevel.INFO, + }); + } + + static interruption({ + started, + type, + reason, + deviceInfo, + }: { + started: boolean; + type: string; + reason: string; + deviceInfo: Partial; + }) { + return new AnalyticsEvent({ + name: `${started ? 'interruption.start' : 'interruption.stop'}`, + level: AnalyticsEventLevel.INFO, + properties: { + reason, + type, + ...deviceInfo, + }, + }); + } + private static eventNameFor(name: string, ok: boolean) { const suffix = ok ? 'success' : 'failed'; return `${name}.${suffix}`; diff --git a/packages/hms-video-store/src/analytics/AnalyticsTransport.ts b/packages/hms-video-store/src/analytics/AnalyticsTransport.ts index 7f3dde0637..77a2955d6e 100644 --- a/packages/hms-video-store/src/analytics/AnalyticsTransport.ts +++ b/packages/hms-video-store/src/analytics/AnalyticsTransport.ts @@ -9,7 +9,30 @@ export abstract class AnalyticsTransport { abstract failedEvents: Queue; private readonly TAG = '[AnalyticsTransport]'; + private eventCount = 0; + private lastResetTime: number = Date.now(); + private readonly MAX_EVENTS_PER_MINUTE: number = 200; + private readonly RESET_INTERVAL_MS: number = 60000; + + private checkRateLimit() { + const now = Date.now(); + if (now - this.lastResetTime >= this.RESET_INTERVAL_MS) { + this.eventCount = 0; + this.lastResetTime = now; + } + if (this.eventCount >= this.MAX_EVENTS_PER_MINUTE) { + throw new Error('Too many events being sent, please check the implementation.'); + } + this.eventCount++; + } + sendEvent(event: AnalyticsEvent) { + try { + this.checkRateLimit(); + } catch (e) { + HMSLogger.w(this.TAG, 'Rate limit exceeded', e); + throw e; + } try { this.sendSingleEvent(event); this.flushFailedEvents(); diff --git a/packages/hms-video-store/src/analytics/stats/BaseStatsAnalytics.ts b/packages/hms-video-store/src/analytics/stats/BaseStatsAnalytics.ts index 001652f464..1a90b47b0d 100644 --- a/packages/hms-video-store/src/analytics/stats/BaseStatsAnalytics.ts +++ b/packages/hms-video-store/src/analytics/stats/BaseStatsAnalytics.ts @@ -21,6 +21,7 @@ import { sleep } from '../../utils/timer-utils'; export abstract class BaseStatsAnalytics { private shouldSendEvent = false; protected sequenceNum = 1; + protected abstract trackAnalytics: Map; constructor( protected store: Store, @@ -38,7 +39,7 @@ export abstract class BaseStatsAnalytics { this.stop(); this.shouldSendEvent = true; this.eventBus.statsUpdate.subscribe(this.handleStatsUpdate.bind(this)); - this.startLoop().catch(e => HMSLogger.e('[StatsAnanlytics]', e.message)); + this.startLoop().catch(e => HMSLogger.e('[StatsAnalytics]', e.message)); } stop = () => { @@ -56,14 +57,39 @@ export abstract class BaseStatsAnalytics { } } - protected abstract sendEvent(): void; + protected sendEvent(): void { + this.trackAnalytics.forEach(trackAnalytic => { + trackAnalytic.clearSamples(); + }); + } + + protected cleanTrackAnalyticsAndCreateSample(shouldCreateSample: boolean) { + // delete track analytics if track is not present in store and no samples are present + this.trackAnalytics.forEach(trackAnalytic => { + if (!this.store.hasTrack(trackAnalytic.track) && !(trackAnalytic.samples.length > 0)) { + this.trackAnalytics.delete(trackAnalytic.track_id); + } + }); + + if (shouldCreateSample) { + this.trackAnalytics.forEach(trackAnalytic => { + trackAnalytic.createSample(); + }); + } + } protected abstract toAnalytics(): PublishAnalyticPayload | SubscribeAnalyticPayload; protected abstract handleStatsUpdate(hmsStats: HMSWebrtcStats): void; } -type TempPublishStats = HMSTrackStats & { availableOutgoingBitrate?: number }; +export type TempStats = HMSTrackStats & { + availableOutgoingBitrate?: number; + calculatedJitterBufferDelay?: number; + avSync?: number; + expectedFrameHeight?: number; + expectedFrameWidth?: number; +}; export abstract class RunningTrackAnalytics { readonly sampleWindowSize: number; @@ -73,9 +99,10 @@ export abstract class RunningTrackAnalytics { ssrc: string; kind: string; rid?: string; - samples: (LocalBaseSample | LocalVideoSample | RemoteAudioSample | RemoteVideoSample)[] = []; - protected tempStats: TempPublishStats[] = []; + samples: (LocalBaseSample | LocalVideoSample | RemoteAudioSample | RemoteVideoSample)[] = []; + protected tempStats: TempStats[] = []; + protected prevLatestStat?: TempStats; constructor({ track, @@ -99,24 +126,35 @@ export abstract class RunningTrackAnalytics { this.sampleWindowSize = sampleWindowSize; } - push(stat: TempPublishStats) { + pushTempStat(stat: TempStats) { this.tempStats.push(stat); + } - if (this.shouldCreateSample()) { - this.samples.push(this.createSample()); - this.tempStats.length = 0; + createSample() { + if (this.tempStats.length === 0) { + return; } + + this.samples.push(this.collateSample()); + this.prevLatestStat = this.getLatestStat(); + this.tempStats.length = 0; + } + + clearSamples() { + this.samples.length = 0; } + abstract shouldCreateSample: () => boolean; + + protected abstract collateSample: () => LocalBaseSample | LocalVideoSample | RemoteAudioSample | RemoteVideoSample; + protected abstract toAnalytics: () => | LocalAudioTrackAnalytics | LocalVideoTrackAnalytics | RemoteAudioTrackAnalytics | RemoteVideoTrackAnalytics; - protected abstract createSample: () => LocalBaseSample | LocalVideoSample | RemoteAudioSample | RemoteVideoSample; - - protected getLatestStat() { + getLatestStat() { return this.tempStats[this.tempStats.length - 1]; } @@ -124,9 +162,7 @@ export abstract class RunningTrackAnalytics { return this.tempStats[0]; } - protected abstract shouldCreateSample: () => boolean; - - protected calculateSum(key: keyof TempPublishStats) { + protected calculateSum(key: keyof TempStats) { const checkStat = this.getLatestStat()[key]; if (typeof checkStat !== 'number') { return; @@ -136,20 +172,25 @@ export abstract class RunningTrackAnalytics { }, 0); } - protected calculateAverage(key: keyof TempPublishStats, round = true) { + protected calculateAverage(key: keyof TempStats, round = true) { const sum = this.calculateSum(key); const avg = sum !== undefined ? sum / this.tempStats.length : undefined; return avg ? (round ? Math.round(avg) : avg) : undefined; } - protected calculateDifferenceForSample(key: keyof TempPublishStats) { - const firstValue = Number(this.tempStats[0][key]) || 0; + protected calculateDifferenceForSample(key: keyof TempStats) { + const firstValue = Number(this.prevLatestStat?.[key]) || 0; const latestValue = Number(this.getLatestStat()[key]) || 0; return latestValue - firstValue; } - protected calculateInstancesOfHigh(key: keyof TempPublishStats, threshold: number) { + protected calculateDifferenceAverage(key: keyof TempStats, round = true) { + const avg = this.calculateDifferenceForSample(key) / this.tempStats.length; + return round ? Math.round(avg) : avg; + } + + protected calculateInstancesOfHigh(key: keyof TempStats, threshold: number) { const checkStat = this.getLatestStat()[key]; if (typeof checkStat !== 'number') { return; @@ -161,10 +202,10 @@ export abstract class RunningTrackAnalytics { } } -export const hasResolutionChanged = (newStat: TempPublishStats, prevStat: TempPublishStats) => +export const hasResolutionChanged = (newStat: TempStats, prevStat: TempStats) => newStat && prevStat && (newStat.frameWidth !== prevStat.frameWidth || newStat.frameHeight !== prevStat.frameHeight); -export const hasEnabledStateChanged = (newStat: TempPublishStats, prevStat: TempPublishStats) => +export const hasEnabledStateChanged = (newStat: TempStats, prevStat: TempStats) => newStat && prevStat && newStat.enabled !== prevStat.enabled; export const removeUndefinedFromObject = >(data: T) => { diff --git a/packages/hms-video-store/src/analytics/stats/PublishStatsAnalytics.ts b/packages/hms-video-store/src/analytics/stats/PublishStatsAnalytics.ts index 87e8470002..4916de08be 100644 --- a/packages/hms-video-store/src/analytics/stats/PublishStatsAnalytics.ts +++ b/packages/hms-video-store/src/analytics/stats/PublishStatsAnalytics.ts @@ -41,21 +41,28 @@ export class PublishStatsAnalytics extends BaseStatsAnalytics { protected sendEvent() { this.eventBus.analytics.publish(AnalyticsEventFactory.publishStats(this.toAnalytics())); + super.sendEvent(); } protected handleStatsUpdate(hmsStats: HMSWebrtcStats) { + let shouldCreateSample = false; + const localTracksStats = hmsStats.getLocalTrackStats(); Object.keys(localTracksStats).forEach(trackIDBeingSent => { const trackStats = localTracksStats[trackIDBeingSent]; const track = this.store.getLocalPeerTracks().find(track => track.getTrackIDBeingSent() === trackIDBeingSent); Object.keys(trackStats).forEach(statId => { const layerStats = trackStats[statId]; - const identifier = track && this.getTrackIdentifier(track?.trackId, layerStats); + if (!track) { + return; + } + const identifier = this.getTrackIdentifier(track.trackId, layerStats); + const newTempStats = { + ...layerStats, + availableOutgoingBitrate: hmsStats.getLocalPeerStats()?.publish?.availableOutgoingBitrate, + }; if (identifier && this.trackAnalytics.has(identifier)) { - this.trackAnalytics.get(identifier)?.push({ - ...layerStats, - availableOutgoingBitrate: hmsStats.getLocalPeerStats()?.publish?.availableOutgoingBitrate, - }); + this.trackAnalytics.get(identifier)?.pushTempStat(newTempStats); } else { if (track) { const trackAnalytics = new RunningLocalTrackAnalytics({ @@ -65,15 +72,19 @@ export class PublishStatsAnalytics extends BaseStatsAnalytics { ssrc: layerStats.ssrc.toString(), kind: layerStats.kind, }); - trackAnalytics.push({ - ...layerStats, - availableOutgoingBitrate: hmsStats.getLocalPeerStats()?.publish?.availableOutgoingBitrate, - }); - this.trackAnalytics.set(this.getTrackIdentifier(track?.trackId, layerStats), trackAnalytics); + trackAnalytics.pushTempStat(newTempStats); + this.trackAnalytics.set(this.getTrackIdentifier(track.trackId, layerStats), trackAnalytics); } } + + const trackAnalytics = this.trackAnalytics.get(identifier); + if (trackAnalytics?.shouldCreateSample()) { + shouldCreateSample = true; + } }); }); + + this.cleanTrackAnalyticsAndCreateSample(shouldCreateSample); } private getTrackIdentifier(trackId: string, stats: HMSTrackStats) { @@ -84,7 +95,7 @@ export class PublishStatsAnalytics extends BaseStatsAnalytics { class RunningLocalTrackAnalytics extends RunningTrackAnalytics { samples: (LocalBaseSample | LocalVideoSample)[] = []; - protected createSample = (): LocalBaseSample | LocalVideoSample => { + protected collateSample = (): LocalBaseSample | LocalVideoSample => { const latestStat = this.getLatestStat(); const qualityLimitationDurations = latestStat.qualityLimitationDurations; @@ -111,8 +122,8 @@ class RunningLocalTrackAnalytics extends RunningTrackAnalytics { avg_available_outgoing_bitrate_bps: this.calculateAverage('availableOutgoingBitrate'), avg_bitrate_bps: this.calculateAverage('bitrate'), avg_fps: this.calculateAverage('framesPerSecond'), - total_packets_lost: this.calculateDifferenceForSample('packetsLost'), - total_packets_sent: this.calculateDifferenceForSample('packetsSent'), + total_packets_lost: this.getLatestStat().packetsLost, + total_packets_sent: this.getLatestStat().packetsSent, total_packet_sent_delay_sec: parseFloat(this.calculateDifferenceForSample('totalPacketSendDelay').toFixed(4)), total_fir_count: this.calculateDifferenceForSample('firCount'), total_pli_count: this.calculateDifferenceForSample('pliCount'), @@ -124,7 +135,7 @@ class RunningLocalTrackAnalytics extends RunningTrackAnalytics { }); }; - protected shouldCreateSample = () => { + shouldCreateSample = () => { const length = this.tempStats.length; const newStat = this.tempStats[length - 1]; const prevStat = this.tempStats[length - 2]; diff --git a/packages/hms-video-store/src/analytics/stats/SubscribeStatsAnalytics.ts b/packages/hms-video-store/src/analytics/stats/SubscribeStatsAnalytics.ts index 9efa0116e4..a1653dff6c 100644 --- a/packages/hms-video-store/src/analytics/stats/SubscribeStatsAnalytics.ts +++ b/packages/hms-video-store/src/analytics/stats/SubscribeStatsAnalytics.ts @@ -4,6 +4,7 @@ import { hasResolutionChanged, removeUndefinedFromObject, RunningTrackAnalytics, + TempStats, } from './BaseStatsAnalytics'; import { RemoteAudioSample, @@ -12,8 +13,10 @@ import { RemoteVideoTrackAnalytics, SubscribeAnalyticPayload, } from './interfaces'; +import { HMSTrackStats } from '../../interfaces'; +import { HMSRemoteVideoTrack } from '../../internal'; import { HMSWebrtcStats } from '../../rtc-stats'; -import { SUBSCRIBE_STATS_SAMPLE_WINDOW } from '../../utils/constants'; +import { MAX_SAFE_INTEGER, SUBSCRIBE_STATS_SAMPLE_WINDOW } from '../../utils/constants'; import AnalyticsEventFactory from '../AnalyticsEventFactory'; export class SubscribeStatsAnalytics extends BaseStatsAnalytics { @@ -40,15 +43,40 @@ export class SubscribeStatsAnalytics extends BaseStatsAnalytics { protected sendEvent() { this.eventBus.analytics.publish(AnalyticsEventFactory.subscribeStats(this.toAnalytics())); + super.sendEvent(); } protected handleStatsUpdate(hmsStats: HMSWebrtcStats) { const remoteTracksStats = hmsStats.getAllRemoteTracksStats(); + let shouldCreateSample = false; Object.keys(remoteTracksStats).forEach(trackID => { - const trackStats = remoteTracksStats[trackID]; const track = this.store.getTrackById(trackID); + const trackStats = remoteTracksStats[trackID]; + const prevTrackStats = this.trackAnalytics.get(trackID)?.getLatestStat(); + + // eslint-disable-next-line complexity + const getCalculatedJitterBufferDelay = (trackStats: HMSTrackStats, prevTrackStats?: TempStats) => { + const prevJBDelay = prevTrackStats?.jitterBufferDelay || 0; + const prevJBEmittedCount = prevTrackStats?.jitterBufferEmittedCount || 0; + const currentJBDelay = (trackStats?.jitterBufferDelay || 0) - prevJBDelay; + const currentJBEmittedCount = (trackStats?.jitterBufferEmittedCount || 0) - prevJBEmittedCount; + + return currentJBEmittedCount > 0 + ? (currentJBDelay * 1000) / currentJBEmittedCount + : prevTrackStats?.calculatedJitterBufferDelay || 0; + }; + + const calculatedJitterBufferDelay = getCalculatedJitterBufferDelay(trackStats, prevTrackStats); + + const avSync = this.calculateAvSyncForStat(trackStats, hmsStats); + const newTempStat: TempStats = { ...trackStats, calculatedJitterBufferDelay, avSync }; + if (trackStats.kind === 'video') { + const definition = (track as HMSRemoteVideoTrack).getPreferredLayerDefinition(); + newTempStat.expectedFrameHeight = definition?.resolution.height; + newTempStat.expectedFrameWidth = definition?.resolution.width; + } if (this.trackAnalytics.has(trackID)) { - this.trackAnalytics.get(trackID)?.push({ ...trackStats }); + this.trackAnalytics.get(trackID)?.pushTempStat(newTempStat); } else { if (track) { const trackAnalytics = new RunningRemoteTrackAnalytics({ @@ -57,52 +85,100 @@ export class SubscribeStatsAnalytics extends BaseStatsAnalytics { ssrc: trackStats.ssrc.toString(), kind: trackStats.kind, }); - trackAnalytics.push({ ...trackStats }); + trackAnalytics.pushTempStat(newTempStat); this.trackAnalytics.set(trackID, trackAnalytics); } } + const trackAnalytics = this.trackAnalytics.get(trackID); + if (trackAnalytics?.shouldCreateSample()) { + shouldCreateSample = true; + } }); + + this.cleanTrackAnalyticsAndCreateSample(shouldCreateSample); + } + + // eslint-disable-next-line complexity + private calculateAvSyncForStat(trackStats: HMSTrackStats, hmsStats: HMSWebrtcStats) { + if (!trackStats.peerID || !trackStats.estimatedPlayoutTimestamp || trackStats.kind !== 'video') { + return; + } + const peer = this.store.getPeerById(trackStats.peerID); + const audioTrack = peer?.audioTrack; + const videoTrack = peer?.videoTrack; + /** + * 1. Send value as MAX_SAFE_INTEGER when either audio or value track is muted for the entire window + * 2. When both audio and video are unmuted for a part of window , then divide the difference by those many number of samples only + */ + const areBothTracksEnabled = audioTrack && videoTrack && audioTrack.enabled && videoTrack.enabled; + if (!areBothTracksEnabled) { + return MAX_SAFE_INTEGER; + } + const audioStats = hmsStats.getRemoteTrackStats(audioTrack.trackId); + if (!audioStats) { + return MAX_SAFE_INTEGER; + } + if (!audioStats.estimatedPlayoutTimestamp) { + return; + } + + // https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats-estimatedplayouttimestamp + return audioStats.estimatedPlayoutTimestamp - trackStats.estimatedPlayoutTimestamp; } } class RunningRemoteTrackAnalytics extends RunningTrackAnalytics { samples: (RemoteAudioSample | RemoteVideoSample)[] = []; - protected createSample = (): RemoteAudioSample | RemoteVideoSample => { + protected collateSample = (): RemoteAudioSample | RemoteVideoSample => { const latestStat = this.getLatestStat(); const firstStat = this.getFirstStat(); const baseSample = { timestamp: Date.now(), - fec_packets_discarded: this.calculateDifferenceForSample('fecPacketsDiscarded'), - fec_packets_received: this.calculateDifferenceForSample('fecPacketsReceived'), - total_samples_duration: this.calculateDifferenceForSample('totalSamplesDuration'), - total_packets_received: this.calculateDifferenceForSample('packetsReceived'), - total_packets_lost: this.calculateDifferenceForSample('packetsLost'), total_pli_count: this.calculateDifferenceForSample('pliCount'), total_nack_count: this.calculateDifferenceForSample('nackCount'), + avg_jitter_buffer_delay: this.calculateAverage('calculatedJitterBufferDelay', false), }; if (latestStat.kind === 'video') { - return removeUndefinedFromObject(baseSample); + return removeUndefinedFromObject({ + ...baseSample, + avg_av_sync_ms: this.calculateAvgAvSyncForSample(), + avg_frames_received_per_sec: this.calculateDifferenceAverage('framesReceived'), + avg_frames_dropped_per_sec: this.calculateDifferenceAverage('framesDropped'), + avg_frames_decoded_per_sec: this.calculateDifferenceAverage('framesDecoded'), + frame_width: this.calculateAverage('frameWidth'), + frame_height: this.calculateAverage('frameHeight'), + expected_frame_width: this.calculateAverage('expectedFrameWidth'), + expected_frame_height: this.calculateAverage('expectedFrameHeight'), + pause_count: this.calculateDifferenceForSample('pauseCount'), + pause_duration_seconds: this.calculateDifferenceForSample('totalPausesDuration'), + freeze_count: this.calculateDifferenceForSample('freezeCount'), + freeze_duration_seconds: this.calculateDifferenceForSample('totalFreezesDuration'), + }); } else { const audio_concealed_samples = (latestStat.concealedSamples || 0) - (latestStat.silentConcealedSamples || 0) - ((firstStat.concealedSamples || 0) - (firstStat.silentConcealedSamples || 0)); - return removeUndefinedFromObject( - Object.assign(baseSample, { - audio_concealed_samples, - audio_level: this.calculateInstancesOfHigh('audioLevel', 0.05), - audio_total_samples_received: this.calculateDifferenceForSample('totalSamplesReceived'), - audio_concealment_events: this.calculateDifferenceForSample('concealmentEvents'), - }), - ); + return removeUndefinedFromObject({ + ...baseSample, + audio_level: this.calculateInstancesOfHigh('audioLevel', 0.05), + audio_concealed_samples, + audio_total_samples_received: this.calculateDifferenceForSample('totalSamplesReceived'), + audio_concealment_events: this.calculateDifferenceForSample('concealmentEvents'), + fec_packets_discarded: this.calculateDifferenceForSample('fecPacketsDiscarded'), + fec_packets_received: this.calculateDifferenceForSample('fecPacketsReceived'), + total_samples_duration: this.calculateDifferenceForSample('totalSamplesDuration'), + total_packets_received: this.calculateDifferenceForSample('packetsReceived'), + total_packets_lost: this.calculateDifferenceForSample('packetsLost'), + }); } }; - protected shouldCreateSample = () => { + shouldCreateSample = () => { const length = this.tempStats.length; const newStat = this.tempStats[length - 1]; const prevStat = this.tempStats[length - 2]; @@ -123,4 +199,15 @@ class RunningRemoteTrackAnalytics extends RunningTrackAnalytics { samples: this.samples, }; }; + + private calculateAvgAvSyncForSample() { + const avSyncValues = this.tempStats.map(stat => stat.avSync); + const validAvSyncValues: number[] = avSyncValues.filter( + (value): value is number => value !== undefined && value !== MAX_SAFE_INTEGER, + ); + if (validAvSyncValues.length === 0) { + return MAX_SAFE_INTEGER; + } + return validAvSyncValues.reduce((a, b) => a + b, 0) / validAvSyncValues.length; + } } diff --git a/packages/hms-video-store/src/analytics/stats/interfaces.ts b/packages/hms-video-store/src/analytics/stats/interfaces.ts index 6d3519ad9b..fd1c52eaf6 100644 --- a/packages/hms-video-store/src/analytics/stats/interfaces.ts +++ b/packages/hms-video-store/src/analytics/stats/interfaces.ts @@ -65,6 +65,7 @@ export interface Resolution { interface RemoteBaseSample { timestamp: number; estimated_playout_timestamp?: number; + avg_jitter_buffer_delay?: number; } export interface RemoteAudioSample extends RemoteBaseSample { @@ -77,7 +78,6 @@ export interface RemoteAudioSample extends RemoteBaseSample { total_samples_duration?: number; total_packets_received?: number; total_packets_lost?: number; - jitter_buffer_delay?: number; jitter_buffer_delay_high_seconds?: number; } @@ -87,4 +87,13 @@ export interface RemoteVideoSample extends RemoteBaseSample { avg_frames_decoded_per_sec?: number; total_pli_count?: number; total_nack_count?: number; + avg_av_sync_ms?: number; + frame_width?: number; + frame_height?: number; + expected_frame_width?: number; + expected_frame_height?: number; + pause_count?: number; + pause_duration_seconds?: number; + freeze_count?: number; + freeze_duration_seconds?: number; } diff --git a/packages/hms-video-store/src/audio-sink-manager/AudioSinkManager.ts b/packages/hms-video-store/src/audio-sink-manager/AudioSinkManager.ts index 0112a670bc..9359b0b88e 100644 --- a/packages/hms-video-store/src/audio-sink-manager/AudioSinkManager.ts +++ b/packages/hms-video-store/src/audio-sink-manager/AudioSinkManager.ts @@ -9,7 +9,6 @@ import { HMSRemoteAudioTrack } from '../media/tracks'; import { HMSRemotePeer } from '../sdk/models/peer'; import { Store } from '../sdk/store'; import HMSLogger from '../utils/logger'; -import { isMobile } from '../utils/support'; import { sleep } from '../utils/timer-utils'; /** @@ -46,6 +45,8 @@ export class AudioSinkManager { this.eventBus.audioTrackRemoved.subscribe(this.handleTrackRemove); this.eventBus.audioTrackUpdate.subscribe(this.handleTrackUpdate); this.eventBus.deviceChange.subscribe(this.handleAudioDeviceChange); + this.eventBus.localVideoUnmutedNatively.subscribe(this.unpauseAudioTracks); + this.eventBus.localAudioUnmutedNatively.subscribe(this.unpauseAudioTracks); } setListener(listener?: HMSUpdateListener) { @@ -97,29 +98,18 @@ export class AudioSinkManager { this.eventBus.audioTrackRemoved.unsubscribe(this.handleTrackRemove); this.eventBus.audioTrackUpdate.unsubscribe(this.handleTrackUpdate); this.eventBus.deviceChange.unsubscribe(this.handleAudioDeviceChange); + this.eventBus.localVideoUnmutedNatively.unsubscribe(this.unpauseAudioTracks); + this.eventBus.localAudioUnmutedNatively.unsubscribe(this.unpauseAudioTracks); this.autoPausedTracks = new Set(); this.state = { ...INITIAL_STATE }; } private handleAudioPaused = async (event: any) => { - const audioEl = event.target as HTMLAudioElement; - //@ts-ignore - const track = audioEl.srcObject?.getAudioTracks()[0]; - if (!track?.enabled) { - // No need to play if already disabled - return; - } - // this means the audio paused because of external factors(headset removal) + // this means the audio paused because of external factors(headset removal, incoming phone call) HMSLogger.d(this.TAG, 'Audio Paused', event.target.id); const audioTrack = this.store.getTrackById(event.target.id); if (audioTrack) { - if (isMobile()) { - // Play after a delay since mobile devices don't call onDevice change event - await sleep(500); - this.playAudioFor(audioTrack as HMSRemoteAudioTrack); - } else { - this.autoPausedTracks.add(audioTrack as HMSRemoteAudioTrack); - } + this.autoPausedTracks.add(audioTrack as HMSRemoteAudioTrack); } }; @@ -148,16 +138,21 @@ export class AudioSinkManager { ); this.eventBus.analytics.publish(AnalyticsEventFactory.audioPlaybackError(ex)); if (audioEl?.error?.code === MediaError.MEDIA_ERR_DECODE) { + // try to wait for main execution to complete first this.removeAudioElement(audioEl, track); await sleep(500); await this.handleTrackAdd({ track, peer, callListener: false }); + if (!this.state.autoplayFailed) { + this.eventBus.analytics.publish( + AnalyticsEventFactory.audioRecovered('Audio recovered after media decode error'), + ); + } } }; track.setAudioElement(audioEl); - track.setVolume(this.volume); + await track.setVolume(this.volume); HMSLogger.d(this.TAG, 'Audio track added', `${track}`); this.init(); // call to create sink element if not already created - await this.autoSelectAudioOutput(); this.audioSink?.append(audioEl); this.outputDevice && (await track.setOutputDevice(this.outputDevice)); audioEl.srcObject = new MediaStream([track.nativeTrack]); @@ -260,39 +255,4 @@ export class AudioSinkManager { track.setAudioElement(null); } }; - - /** - * Mweb is not able to play via call channel by default, this is to switch from media channel to call channel - */ - // eslint-disable-next-line complexity - private autoSelectAudioOutput = async () => { - if (this.audioSink?.children.length === 0) { - let bluetoothDevice: InputDeviceInfo | null = null; - let speakerPhone: InputDeviceInfo | null = null; - let wired: InputDeviceInfo | null = null; - let earpiece: InputDeviceInfo | null = null; - - for (const device of this.deviceManager.audioInput) { - if (device.label.toLowerCase().includes('speakerphone')) { - speakerPhone = device; - } - if (device.label.toLowerCase().includes('wired')) { - wired = device; - } - if (device.label.toLowerCase().includes('bluetooth')) { - bluetoothDevice = device; - } - if (device.label.toLowerCase().includes('earpiece')) { - earpiece = device; - } - } - const localAudioTrack = this.store.getLocalPeer()?.audioTrack; - if (localAudioTrack && earpiece) { - await localAudioTrack.setSettings({ deviceId: earpiece?.deviceId }); - await localAudioTrack.setSettings({ - deviceId: bluetoothDevice?.deviceId || wired?.deviceId || speakerPhone?.deviceId, - }); - } - } - }; } diff --git a/packages/hms-video-store/src/common/PluginUsageTracker.ts b/packages/hms-video-store/src/common/PluginUsageTracker.ts new file mode 100644 index 0000000000..ee959abec6 --- /dev/null +++ b/packages/hms-video-store/src/common/PluginUsageTracker.ts @@ -0,0 +1,53 @@ +import AnalyticsEvent from '../analytics/AnalyticsEvent'; +import { EventBus } from '../events/EventBus'; + +export class PluginUsageTracker { + private pluginUsage: Map = new Map(); + private pluginLastAddedAt: Map = new Map(); + + constructor(private eventBus: EventBus) { + this.eventBus.analytics.subscribe(e => this.updatePluginUsageData(e)); + } + + getPluginUsage = (name: string) => { + if (!this.pluginUsage.has(name)) { + this.pluginUsage.set(name, 0); + } + if (this.pluginLastAddedAt.has(name)) { + const lastAddedAt = this.pluginLastAddedAt.get(name) || 0; + const extraDuration = lastAddedAt ? Date.now() - lastAddedAt : 0; + this.pluginUsage.set(name, (this.pluginUsage.get(name) || 0) + extraDuration); + this.pluginLastAddedAt.delete(name); + } + const finalValue = this.pluginUsage.get(name); + return finalValue; + }; + + // eslint-disable-next-line complexity + updatePluginUsageData = (event: AnalyticsEvent) => { + const name = event.properties?.plugin_name || ''; + switch (event.name) { + case 'mediaPlugin.toggled.on': + case 'mediaPlugin.added': { + const addedAt = event.properties.added_at || Date.now(); + this.pluginLastAddedAt.set(name, addedAt); + break; + } + case 'mediaPlugin.toggled.off': + case 'mediaPlugin.stats': { + if (this.pluginLastAddedAt.has(name)) { + const duration = event.properties.duration || (Date.now() - (this.pluginLastAddedAt.get(name) || 0)) / 1000; + this.pluginUsage.set(name, (this.pluginUsage.get(name) || 0) + Math.max(duration, 0) * 1000); + this.pluginLastAddedAt.delete(name); + } + break; + } + default: + } + }; + + cleanup = () => { + this.pluginLastAddedAt.clear(); + this.pluginUsage.clear(); + }; +} diff --git a/packages/hms-video-store/src/connection/HMSConnection.ts b/packages/hms-video-store/src/connection/HMSConnection.ts index 3e4fc3b771..198b02eb36 100644 --- a/packages/hms-video-store/src/connection/HMSConnection.ts +++ b/packages/hms-video-store/src/connection/HMSConnection.ts @@ -1,6 +1,8 @@ +import IConnectionObserver, { RTCIceCandidatePair } from './IConnectionObserver'; import { HMSConnectionRole } from './model'; import { ErrorFactory } from '../error/ErrorFactory'; import { HMSAction } from '../error/HMSAction'; +import { HMSAudioTrackSettings, HMSVideoTrackSettings } from '../media/settings'; import { HMSLocalTrack, HMSLocalVideoTrack } from '../media/tracks'; import { TrackState } from '../notification-manager'; import JsonRpcSignal from '../signal/jsonrpc'; @@ -8,15 +10,12 @@ import HMSLogger from '../utils/logger'; import { enableOpusDtx, fixMsid } from '../utils/session-description'; const TAG = '[HMSConnection]'; -interface RTCIceCandidatePair { - local: RTCIceCandidate; - remote: RTCIceCandidate; -} export default abstract class HMSConnection { readonly role: HMSConnectionRole; protected readonly signal: JsonRpcSignal; + protected abstract readonly observer: IConnectionObserver; abstract readonly nativeConnection: RTCPeerConnection; /** * We keep a list of pending IceCandidates received @@ -29,6 +28,8 @@ export default abstract class HMSConnection { * - [HMSSubscribeConnection] clears this list as soon as we call [addIceCandidate] */ readonly candidates = new Array(); + // @ts-ignore + sfuNodeId?: string; selectedCandidatePair?: RTCIceCandidatePair; @@ -49,6 +50,10 @@ export default abstract class HMSConnection { return this.role === HMSConnectionRole.Publish ? HMSAction.PUBLISH : HMSAction.SUBSCRIBE; } + setSfuNodeId(nodeId?: string) { + this.sfuNodeId = nodeId; + } + addTransceiver(track: MediaStreamTrack, init: RTCRtpTransceiverInit): RTCRtpTransceiver { return this.nativeConnection.addTransceiver(track, init); } @@ -108,7 +113,7 @@ export default abstract class HMSConnection { return this.nativeConnection.getSenders(); } - logSelectedIceCandidatePairs() { + handleSelectedIceCandidatePairs() { /** * for the very first peer in the room we don't have any subscribe ice candidates * because the peer hasn't subscribed to anything. @@ -125,26 +130,29 @@ export default abstract class HMSConnection { if (transmitter.transport) { const iceTransport = transmitter.transport.iceTransport; - const logSelectedCandidate = () => { + const handleSelectedCandidate = () => { // @ts-expect-error if (typeof iceTransport.getSelectedCandidatePair === 'function') { // @ts-expect-error this.selectedCandidatePair = iceTransport.getSelectedCandidatePair(); - HMSLogger.d( - TAG, - `${HMSConnectionRole[this.role]} connection`, - `selected ${kindOfTrack || 'unknown'} candidate pair`, - JSON.stringify(this.selectedCandidatePair, null, 2), - ); + if (this.selectedCandidatePair) { + this.observer.onSelectedCandidatePairChange(this.selectedCandidatePair); + HMSLogger.d( + TAG, + `${HMSConnectionRole[this.role]} connection`, + `selected ${kindOfTrack || 'unknown'} candidate pair`, + JSON.stringify(this.selectedCandidatePair, null, 2), + ); + } } }; // @ts-expect-error if (typeof iceTransport.onselectedcandidatepairchange === 'function') { // @ts-expect-error - iceTransport.onselectedcandidatepairchange = logSelectedCandidate; + iceTransport.onselectedcandidatepairchange = handleSelectedCandidate; } - logSelectedCandidate(); + handleSelectedCandidate(); } }); } catch (error) { @@ -162,8 +170,12 @@ export default abstract class HMSConnection { } } - async setMaxBitrateAndFramerate(track: HMSLocalTrack) { - const maxBitrate = track.settings.maxBitrate; + // eslint-disable-next-line + async setMaxBitrateAndFramerate( + track: HMSLocalTrack, + updatedSettings?: HMSAudioTrackSettings | HMSVideoTrackSettings, + ) { + const maxBitrate = updatedSettings?.maxBitrate || track.settings.maxBitrate; const maxFramerate = track instanceof HMSLocalVideoTrack && track.settings.maxFramerate; const sender = this.getSenders().find(s => s?.track?.id === track.getTrackIDBeingSent()); @@ -192,7 +204,7 @@ export default abstract class HMSConnection { return await this.nativeConnection.getStats(); } - async close() { + close() { this.nativeConnection.close(); } diff --git a/packages/hms-video-store/src/connection/IConnectionObserver.ts b/packages/hms-video-store/src/connection/IConnectionObserver.ts index fb897805f0..e109465d57 100644 --- a/packages/hms-video-store/src/connection/IConnectionObserver.ts +++ b/packages/hms-video-store/src/connection/IConnectionObserver.ts @@ -1,6 +1,15 @@ +export interface RTCIceCandidatePair { + local?: RTCIceCandidate; + remote?: RTCIceCandidate; +} + export default interface IConnectionObserver { onIceConnectionChange(newState: RTCIceConnectionState): void; // @TODO(eswar): Remove this. Use iceconnectionstate change with interval and threshold. onConnectionStateChange(newState: RTCPeerConnectionState): void; + + onIceCandidate(candidate: RTCIceCandidate): void; + + onSelectedCandidatePairChange(candidatePair: RTCIceCandidatePair): void; } diff --git a/packages/hms-video-store/src/connection/publish/publishConnection.ts b/packages/hms-video-store/src/connection/publish/publishConnection.ts index 8162ac2809..422c7eb8c8 100644 --- a/packages/hms-video-store/src/connection/publish/publishConnection.ts +++ b/packages/hms-video-store/src/connection/publish/publishConnection.ts @@ -7,7 +7,7 @@ import { HMSConnectionRole } from '../model'; export default class HMSPublishConnection extends HMSConnection { private readonly TAG = '[HMSPublishConnection]'; - private readonly observer: IPublishConnectionObserver; + protected readonly observer: IPublishConnectionObserver; readonly nativeConnection: RTCPeerConnection; readonly channel: RTCDataChannel; @@ -23,6 +23,7 @@ export default class HMSPublishConnection extends HMSConnection { this.nativeConnection.onicecandidate = ({ candidate }) => { if (candidate) { + this.observer.onIceCandidate(candidate); signal.trickle(this.role, candidate); } }; @@ -31,7 +32,6 @@ export default class HMSPublishConnection extends HMSConnection { this.observer.onIceConnectionChange(this.nativeConnection.iceConnectionState); }; - // @TODO(eswar): Remove this. Use iceconnectionstate change with interval and threshold. this.nativeConnection.onconnectionstatechange = () => { this.observer.onConnectionStateChange(this.nativeConnection.connectionState); @@ -50,6 +50,11 @@ export default class HMSPublishConnection extends HMSConnection { }; } + close() { + super.close(); + this.channel.close(); + } + initAfterJoin() { this.nativeConnection.onnegotiationneeded = async () => { HMSLogger.d(this.TAG, `onnegotiationneeded`); diff --git a/packages/hms-video-store/src/connection/subscribe/subscribeConnection.ts b/packages/hms-video-store/src/connection/subscribe/subscribeConnection.ts index b394bb843a..9120824a24 100644 --- a/packages/hms-video-store/src/connection/subscribe/subscribeConnection.ts +++ b/packages/hms-video-store/src/connection/subscribe/subscribeConnection.ts @@ -18,7 +18,7 @@ import { HMSConnectionRole } from '../model'; export default class HMSSubscribeConnection extends HMSConnection { private readonly TAG = '[HMSSubscribeConnection]'; private readonly remoteStreams = new Map(); - private readonly observer: ISubscribeConnectionObserver; + protected readonly observer: ISubscribeConnectionObserver; private readonly MAX_RETRIES = 3; readonly nativeConnection: RTCPeerConnection; @@ -60,6 +60,7 @@ export default class HMSSubscribeConnection extends HMSConnection { this.nativeConnection.onicecandidate = e => { if (e.candidate !== null) { + this.observer.onIceCandidate(e.candidate); this.signal.trickle(this.role, e.candidate); } }; @@ -97,8 +98,11 @@ export default class HMSSubscribeConnection extends HMSConnection { }); const remote = this.remoteStreams.get(streamId)!; - const TrackCls = e.track.kind === 'audio' ? HMSRemoteAudioTrack : HMSRemoteVideoTrack; - const track = new TrackCls(remote, e.track); + const isAudioTrack = e.track.kind === 'audio'; + const TrackCls = isAudioTrack ? HMSRemoteAudioTrack : HMSRemoteVideoTrack; + const track = isAudioTrack + ? new TrackCls(remote, e.track) + : new TrackCls(remote, e.track, undefined, this.isFlagEnabled(InitFlags.FLAG_DISABLE_NONE_LAYER_REQUEST)); // reset the simulcast layer to none when new video tracks are added, UI will subscribe when required if (e.track.kind === 'video') { remote.setVideoLayerLocally(HMSSimulcastLayer.NONE, 'addTrack', 'subscribeConnection'); @@ -153,8 +157,8 @@ export default class HMSSubscribeConnection extends HMSConnection { return this.sendMessage(request, id); } - async close() { - await super.close(); + close() { + super.close(); this.apiChannel?.close(); } diff --git a/packages/hms-video-store/src/device-manager/DeviceManager.ts b/packages/hms-video-store/src/device-manager/DeviceManager.ts index 2feb3481b5..8fedbc647c 100644 --- a/packages/hms-video-store/src/device-manager/DeviceManager.ts +++ b/packages/hms-video-store/src/device-manager/DeviceManager.ts @@ -4,8 +4,10 @@ import { ErrorFactory } from '../error/ErrorFactory'; import { HMSException } from '../error/HMSException'; import { EventBus } from '../events/EventBus'; import { DeviceMap, HMSDeviceChangeEvent, SelectedDevices } from '../interfaces'; +import { getAudioDeviceCategory, HMSAudioDeviceCategory, isIOS } from '../internal'; import { HMSAudioTrackSettingsBuilder, HMSVideoTrackSettingsBuilder } from '../media/settings'; import { HMSLocalAudioTrack, HMSLocalTrack, HMSLocalVideoTrack } from '../media/tracks'; +import { HMSTrackExceptionTrackType } from '../media/tracks/HMSTrackExceptionTrackType'; import { Store } from '../sdk/store'; import HMSLogger from '../utils/logger'; import { debounce } from '../utils/timer-utils'; @@ -29,10 +31,18 @@ export class DeviceManager implements HMSDeviceManager { hasWebcamPermission = false; hasMicrophonePermission = false; + currentSelection: SelectedDevices = { + audioInput: undefined, + videoInput: undefined, + audioOutput: undefined, + }; + private readonly TAG = '[Device Manager]:'; private initialized = false; private videoInputChanged = false; private audioInputChanged = false; + private earpieceSelected = false; + private timer: ReturnType | null = null; constructor(private store: Store, private eventBus: EventBus) { const isLocalTrackEnabled = ({ enabled, track }: { enabled: boolean; track: HMSLocalTrack }) => @@ -86,25 +96,33 @@ export class DeviceManager implements HMSDeviceManager { return newDevice; }; - async init(force = false) { + async init(force = false, logAnalytics = true) { if (this.initialized && !force) { return; } !this.initialized && navigator.mediaDevices.addEventListener('devicechange', this.handleDeviceChange); this.initialized = true; await this.enumerateDevices(); + // do it only on initial load. + if (!force) { + await this.updateToActualDefaultDevice(); + await this.autoSelectAudioOutput(); + this.startPollingForDevices(); + } this.logDevices('Init'); await this.setOutputDevice(); this.eventBus.deviceChange.publish({ devices: this.getDevices(), } as HMSDeviceChangeEvent); - this.eventBus.analytics.publish( - AnalyticsEventFactory.deviceChange({ - selection: this.getCurrentSelection(), - type: 'list', - devices: this.getDevices(), - }), - ); + if (logAnalytics) { + this.eventBus.analytics.publish( + AnalyticsEventFactory.deviceChange({ + selection: this.getCurrentSelection(), + type: 'list', + devices: this.getDevices(), + }), + ); + } } getDevices(): DeviceMap { @@ -117,6 +135,11 @@ export class DeviceManager implements HMSDeviceManager { cleanup() { this.initialized = false; + this.earpieceSelected = false; + if (this.timer) { + clearTimeout(this.timer); + this.timer = null; + } this.audioInput = []; this.audioOutput = []; this.videoInput = []; @@ -186,15 +209,36 @@ export class DeviceManager implements HMSDeviceManager { } }; + /** + * For example, if a different device, say OBS is selected as default from chrome settings, when you do getUserMedia with default, that is not the device + * you get. So update to the browser settings default device + * Update only when initial deviceId is not passed + */ + private updateToActualDefaultDevice = async () => { + const localPeer = this.store.getLocalPeer(); + const videoDeviceId = this.store.getConfig()?.settings?.videoDeviceId; + if (!videoDeviceId && localPeer?.videoTrack) { + await localPeer.videoTrack.setSettings({ deviceId: this.videoInput[0]?.deviceId }, true); + } + const audioDeviceId = this.store.getConfig()?.settings?.audioInputDeviceId; + if (!audioDeviceId && localPeer?.audioTrack) { + const getInitialDeviceId = () => { + const nonIPhoneDevice = this.audioInput.find(device => !device.label.toLowerCase().includes('iphone')); + return isIOS() && nonIPhoneDevice ? nonIPhoneDevice?.deviceId : this.getNewAudioInputDevice()?.deviceId; + }; + const deviceIdToUpdate = getInitialDeviceId(); + if (deviceIdToUpdate) { + await localPeer.audioTrack.setSettings({ deviceId: getInitialDeviceId() }, true); + } + } + }; + private handleDeviceChange = debounce(async () => { await this.enumerateDevices(); this.logDevices('After Device Change'); const localPeer = this.store.getLocalPeer(); - const audioTrack = localPeer?.audioTrack; await this.setOutputDevice(true); - if (audioTrack) { - await this.handleAudioInputDeviceChange(localPeer?.audioTrack); - } + await this.handleAudioInputDeviceChange(localPeer?.audioTrack); await this.handleVideoInputDeviceChange(localPeer?.videoTrack); this.eventBus.analytics.publish( AnalyticsEventFactory.deviceChange({ @@ -212,16 +256,12 @@ export class DeviceManager implements HMSDeviceManager { * @returns {MediaDeviceInfo} */ getNewAudioInputDevice() { - const localPeer = this.store.getLocalPeer(); - const audioTrack = localPeer?.audioTrack; - const manualSelection = this.audioInput.find( - device => device.deviceId === audioTrack?.getManuallySelectedDeviceId(), - ); + const manualSelection = this.getManuallySelectedAudioDevice(); if (manualSelection) { return manualSelection; } // if manually selected device is not available, reset on the track - audioTrack?.resetManuallySelectedDeviceId(); + this.store.getLocalPeer()?.audioTrack?.resetManuallySelectedDeviceId(); const defaultDevice = this.audioInput.find(device => device.deviceId === 'default'); if (defaultDevice) { // Selecting a non-default device so that the deviceId comparision does not give @@ -292,7 +332,7 @@ export class DeviceManager implements HMSDeviceManager { this.eventBus.analytics.publish( AnalyticsEventFactory.deviceChange({ selection: { audioInput: newSelection }, - error: ErrorFactory.TracksErrors.SelectedDeviceMissing('audio'), + error: ErrorFactory.TracksErrors.SelectedDeviceMissing(HMSTrackExceptionTrackType.AUDIO), devices: this.getDevices(), type: 'audioInput', }), @@ -306,6 +346,7 @@ export class DeviceManager implements HMSDeviceManager { .codec(settings.codec) .maxBitrate(settings.maxBitrate) .deviceId(newSelection.deviceId) + .audioMode(settings.audioMode) .build(); try { await audioTrack.setSettings(newAudioTrackSettings, true); @@ -349,7 +390,7 @@ export class DeviceManager implements HMSDeviceManager { this.eventBus.analytics.publish( AnalyticsEventFactory.deviceChange({ selection: { videoInput: newSelection }, - error: ErrorFactory.TracksErrors.SelectedDeviceMissing('video'), + error: ErrorFactory.TracksErrors.SelectedDeviceMissing(HMSTrackExceptionTrackType.VIDEO), devices: this.getDevices(), type: 'video', }), @@ -367,7 +408,7 @@ export class DeviceManager implements HMSDeviceManager { .deviceId(newSelection.deviceId) .build(); try { - await (videoTrack as HMSLocalVideoTrack).setSettings(newVideoTrackSettings, true); + await videoTrack.setSettings(newVideoTrackSettings, true); // On replace track, enabled will be true. Need to be set to previous state // videoTrack.setEnabled(enabled); // TODO: remove this once verified on qa. this.eventBus.deviceChange.publish({ @@ -395,6 +436,86 @@ export class DeviceManager implements HMSDeviceManager { } }; + getManuallySelectedAudioDevice() { + const localPeer = this.store.getLocalPeer(); + const audioTrack = localPeer?.audioTrack; + return this.audioInput.find(device => device.deviceId === audioTrack?.getManuallySelectedDeviceId()); + } + + // specifically used for mweb + categorizeAudioInputDevices() { + let bluetoothDevice: InputDeviceInfo | null = null; + let speakerPhone: InputDeviceInfo | null = null; + let wired: InputDeviceInfo | null = null; + let earpiece: InputDeviceInfo | null = null; + + for (const device of this.audioInput) { + const deviceCategory = getAudioDeviceCategory(device.label); + if (deviceCategory === HMSAudioDeviceCategory.SPEAKERPHONE) { + speakerPhone = device; + } else if (deviceCategory === HMSAudioDeviceCategory.WIRED) { + wired = device; + } else if (deviceCategory === HMSAudioDeviceCategory.BLUETOOTH) { + bluetoothDevice = device; + } else if (deviceCategory === HMSAudioDeviceCategory.EARPIECE) { + earpiece = device; + } + } + + return { bluetoothDevice, speakerPhone, wired, earpiece }; + } + + private startPollingForDevices = () => { + // device change supported, no polling needed + if ('ondevicechange' in navigator.mediaDevices) { + return; + } + this.timer = setTimeout(() => { + (async () => { + await this.enumerateDevices(); + await this.autoSelectAudioOutput(); + this.startPollingForDevices(); + })(); + }, 5000); + }; + + /** + * Mweb is not able to play via call channel by default, this is to switch from media channel to call channel + */ + // eslint-disable-next-line complexity + private autoSelectAudioOutput = async () => { + if ('ondevicechange' in navigator.mediaDevices) { + return; + } + const { bluetoothDevice, earpiece, speakerPhone, wired } = this.categorizeAudioInputDevices(); + const localAudioTrack = this.store.getLocalPeer()?.audioTrack; + if (!localAudioTrack || !earpiece) { + return; + } + const manualSelection = this.getManuallySelectedAudioDevice(); + const externalDeviceID = + manualSelection?.deviceId || bluetoothDevice?.deviceId || wired?.deviceId || speakerPhone?.deviceId; + HMSLogger.d(this.TAG, 'externalDeviceID', externalDeviceID); + // already selected appropriate device + if (localAudioTrack.settings.deviceId === externalDeviceID && this.earpieceSelected) { + return; + } + if (!this.earpieceSelected) { + if (bluetoothDevice?.deviceId === externalDeviceID) { + this.earpieceSelected = true; + return; + } + await localAudioTrack.setSettings({ deviceId: earpiece?.deviceId }, true); + this.earpieceSelected = true; + } + await localAudioTrack.setSettings( + { + deviceId: externalDeviceID, + }, + true, + ); + }; + // eslint-disable-next-line complexity private getAudioOutputDeviceMatchingInput(inputDevice?: MediaDeviceInfo) { const blacklist = this.store.getConfig()?.settings?.speakerAutoSelectionBlacklist || []; diff --git a/packages/hms-video-store/src/diagnostics/CQSCalculator.ts b/packages/hms-video-store/src/diagnostics/CQSCalculator.ts new file mode 100644 index 0000000000..35e36c83e6 --- /dev/null +++ b/packages/hms-video-store/src/diagnostics/CQSCalculator.ts @@ -0,0 +1,33 @@ +export class CQSCalculator { + private networkScores: number[] = []; + private lastPushedAt = 0; + + pushScore(score?: number) { + if (!score || score < 0) { + return; + } + + if (this.networkScores.length === 0) { + this.networkScores.push(score); + this.lastPushedAt = Date.now(); + } else { + this.addPendingCQSTillNow(); + } + } + + addPendingCQSTillNow() { + if (this.networkScores.length > 0) { + let timeDiffInSec = (Date.now() - this.lastPushedAt) / 1000; + while (timeDiffInSec > 0) { + this.networkScores.push(this.networkScores[this.networkScores.length - 1]); + timeDiffInSec -= 1; + } + + this.lastPushedAt = Date.now(); + } + } + + getCQS() { + return this.networkScores.reduce((acc, score) => acc + score, 0) / this.networkScores.length; + } +} diff --git a/packages/hms-video-store/src/diagnostics/ConnectivityCheck.ts b/packages/hms-video-store/src/diagnostics/ConnectivityCheck.ts new file mode 100644 index 0000000000..ca5a2353bd --- /dev/null +++ b/packages/hms-video-store/src/diagnostics/ConnectivityCheck.ts @@ -0,0 +1,215 @@ +import { CONNECTIVITY_TEST_DURATION } from './constants'; +import { CQSCalculator } from './CQSCalculator'; +import { DiagnosticsStatsCollector } from './DiagnosticsStatsCollector'; +import { ConnectivityCheckResult, ConnectivityState, HMSDiagnosticsConnectivityListener } from './interfaces'; +import { RTCIceCandidatePair } from '../connection/IConnectionObserver'; +import { + HMSConnectionQuality, + HMSException, + HMSRoom, + HMSTrack, + HMSTrackType, + HMSTrackUpdate, + HMSUpdateListener, +} from '../internal'; +import { HMSSdk } from '../sdk'; +import { HMSPeer } from '../sdk/models/peer'; + +export class ConnectivityCheck implements HMSDiagnosticsConnectivityListener { + private wsConnected = false; + private websocketURL?: string; + private initConnected = false; + + private isPublishICEConnected = false; + private isSubscribeICEConnected = false; + private selectedPublishICECandidate?: RTCIceCandidatePair; + private selectedSubscribeICECandidate?: RTCIceCandidatePair; + private gatheredPublishICECandidates: RTCIceCandidate[] = []; + private gatheredSubscribeICECandidates: RTCIceCandidate[] = []; + private errors: HMSException[] = []; + private isAudioTrackCaptured = false; + private isVideoTrackCaptured = false; + private isAudioTrackPublished = false; + private isVideoTrackPublished = false; + private statsCollector: DiagnosticsStatsCollector; + private cqsCalculator = new CQSCalculator(); + + private cleanupTimer?: number; + private timestamp = Date.now(); + private _state?: ConnectivityState; + private get state(): ConnectivityState | undefined { + return this._state; + } + private set state(value: ConnectivityState | undefined) { + if (value === undefined || (this._state !== undefined && value < this._state)) { + return; + } + this._state = value; + this.progressCallback?.(value); + } + + constructor( + private sdk: HMSSdk, + private sdkListener: HMSUpdateListener, + private progressCallback: (state: ConnectivityState) => void, + private completionCallback: (state: ConnectivityCheckResult) => void, + ) { + this.statsCollector = new DiagnosticsStatsCollector(sdk); + this.state = ConnectivityState.STARTING; + } + onRoomUpdate = this.sdkListener.onRoomUpdate.bind(this.sdkListener); + onPeerUpdate = this.sdkListener.onPeerUpdate.bind(this.sdkListener); + onMessageReceived = this.sdkListener.onMessageReceived.bind(this.sdkListener); + onReconnected = this.sdkListener.onReconnected.bind(this.sdkListener); + onRoleChangeRequest = this.sdkListener.onRoleChangeRequest.bind(this.sdkListener); + onRoleUpdate = this.sdkListener.onRoleUpdate.bind(this.sdkListener); + onChangeTrackStateRequest = this.sdkListener.onChangeTrackStateRequest.bind(this.sdkListener); + onChangeMultiTrackStateRequest = this.sdkListener.onChangeMultiTrackStateRequest.bind(this.sdkListener); + onRemovedFromRoom = this.sdkListener.onRemovedFromRoom.bind(this.sdkListener); + onNetworkQuality = this.sdkListener.onNetworkQuality?.bind(this.sdkListener); + onPreview = this.sdkListener.onPreview.bind(this.sdkListener); + onDeviceChange = this.sdkListener.onDeviceChange?.bind(this.sdkListener); + onSessionStoreUpdate = this.sdkListener.onSessionStoreUpdate.bind(this.sdkListener); + onPollsUpdate = this.sdkListener.onPollsUpdate.bind(this.sdkListener); + onWhiteboardUpdate = this.sdkListener.onWhiteboardUpdate.bind(this.sdkListener); + + handleConnectionQualityUpdate = (qualities: HMSConnectionQuality[]) => { + const localPeerQuality = qualities.find(quality => quality.peerID === this.sdk?.store.getLocalPeer()?.peerId); + this.cqsCalculator.pushScore(localPeerQuality?.downlinkQuality); + }; + + onICESuccess(isPublish: boolean): void { + if (isPublish) { + this.isPublishICEConnected = true; + } else { + this.isSubscribeICEConnected = true; + } + + if (this.isPublishICEConnected && this.isSubscribeICEConnected) { + this.state = ConnectivityState.ICE_ESTABLISHED; + } + } + + onSelectedICECandidatePairChange(candidatePair: RTCIceCandidatePair, isPublish: boolean): void { + if (isPublish) { + this.selectedPublishICECandidate = candidatePair; + } else { + this.selectedSubscribeICECandidate = candidatePair; + } + } + + onICECandidate(candidate: RTCIceCandidate, isPublish: boolean): void { + if (isPublish) { + this.gatheredPublishICECandidates.push(candidate); + } else { + this.gatheredSubscribeICECandidates.push(candidate); + } + } + + onMediaPublished(track: HMSTrack): void { + switch (track.type) { + case HMSTrackType.AUDIO: + this.isAudioTrackPublished = true; + break; + case HMSTrackType.VIDEO: + this.isVideoTrackPublished = true; + break; + default: + break; + } + + if (this.isVideoTrackPublished && this.isAudioTrackPublished) { + this.state = ConnectivityState.MEDIA_PUBLISHED; + } + } + + onInitSuccess(websocketURL: string): void { + this.websocketURL = websocketURL; + this.initConnected = true; + this.state = ConnectivityState.INIT_FETCHED; + } + + onSignallingSuccess(): void { + this.wsConnected = true; + this.state = ConnectivityState.SIGNAL_CONNECTED; + } + + onJoin(room: HMSRoom): void { + this.sdkListener.onJoin(room); + this.sdk.getWebrtcInternals()?.onStatsChange(stats => this.statsCollector.handleStatsUpdate(stats)); + this.sdk.getWebrtcInternals()?.start(); + this.cleanupTimer = window.setTimeout(() => { + this.cleanupAndReport(); + }, CONNECTIVITY_TEST_DURATION); + } + + onError(error: HMSException): void { + this.sdkListener.onError(error); + this.errors.push(error); + if (error?.isTerminal) { + this.cleanupAndReport(); + } + } + + // eslint-disable-next-line complexity + onTrackUpdate(type: HMSTrackUpdate, track: HMSTrack, peer: HMSPeer): void { + this.sdkListener.onTrackUpdate(type, track, peer); + if (peer.isLocal && type === HMSTrackUpdate.TRACK_ADDED) { + switch (track.type) { + case HMSTrackType.AUDIO: + this.isAudioTrackCaptured = true; + break; + case HMSTrackType.VIDEO: + this.isVideoTrackCaptured = true; + break; + default: + break; + } + + if (this.isVideoTrackCaptured && this.isAudioTrackCaptured) { + this.state = ConnectivityState.MEDIA_CAPTURED; + } + } + } + + onReconnecting(error: HMSException): void { + this.sdkListener.onReconnecting(error); + this.cqsCalculator.addPendingCQSTillNow(); + } + + cleanupAndReport() { + clearTimeout(this.cleanupTimer); + this.cleanupTimer = undefined; + if (this.state === ConnectivityState.MEDIA_PUBLISHED) { + this.state = ConnectivityState.COMPLETED; + } + this.completionCallback?.(this.buildReport()); + this.sdk.leave(); + } + + private buildReport(): ConnectivityCheckResult { + this.cqsCalculator.addPendingCQSTillNow(); + const connectionQualityScore = this.cqsCalculator.getCQS(); + const stats = this.statsCollector.buildReport(); + return { + testTimestamp: this.timestamp, + connectivityState: this.state, + errors: this.errors, + signallingReport: { + isConnected: this.wsConnected, + isInitConnected: this.initConnected, + websocketUrl: this.websocketURL, + }, + mediaServerReport: { + stats, + connectionQualityScore, + isPublishICEConnected: this.isPublishICEConnected, + isSubscribeICEConnected: this.isSubscribeICEConnected, + publishICECandidatePairSelected: this.selectedPublishICECandidate, + subscribeICECandidatePairSelected: this.selectedSubscribeICECandidate, + publishIceCandidatesGathered: this.gatheredPublishICECandidates, + subscribeIceCandidatesGathered: this.gatheredSubscribeICECandidates, + }, + }; + } +} diff --git a/packages/hms-video-store/src/diagnostics/DiagnosticsStatsCollector.ts b/packages/hms-video-store/src/diagnostics/DiagnosticsStatsCollector.ts new file mode 100644 index 0000000000..af12183fae --- /dev/null +++ b/packages/hms-video-store/src/diagnostics/DiagnosticsStatsCollector.ts @@ -0,0 +1,113 @@ +import { DiagnosticsRTCStatsReport } from '.'; +import { HMSPeerStats, HMSTrackStats } from '../interfaces'; +import { HMSWebrtcStats } from '../rtc-stats'; +import { computeBitrate } from '../rtc-stats/utils'; +import { HMSSdk } from '../sdk'; + +const isValidNumber = (num: number | undefined): boolean => !!num && !isNaN(num); +const getLastElement = (arr: T[]): T | undefined => arr[arr.length - 1]; +const calculateAverage = (arr: T[], predicate: (val: T) => number | undefined): number => { + const filteredArr = arr.filter(curr => isValidNumber(predicate(curr))); + return filteredArr.reduce((acc, curr) => acc + (predicate(curr) || 0), 0) / filteredArr.length; +}; + +export class DiagnosticsStatsCollector { + private peerStatsList: HMSPeerStats[] = []; + private localAudioTrackStatsList: Record[] = []; + private localVideoTrackStatsList: Record[] = []; + private remoteAudioTrackStatsList: HMSTrackStats[] = []; + private remoteVideoTrackStatsList: HMSTrackStats[] = []; + + constructor(private sdk: HMSSdk) {} + + async handleStatsUpdate(stats: HMSWebrtcStats) { + const localPeerStats = stats.getLocalPeerStats(); + if (localPeerStats) { + this.peerStatsList.push(localPeerStats); + } + + const localAudioTrackID = this.sdk.getLocalPeer()?.audioTrack?.nativeTrack?.id; + const localVideoTrackID = this.sdk.getLocalPeer()?.videoTrack?.nativeTrack?.id; + + const localTrackStats = stats.getLocalTrackStats(); + if (localTrackStats) { + localAudioTrackID && this.localAudioTrackStatsList.push(localTrackStats[localAudioTrackID]); + localVideoTrackID && this.localVideoTrackStatsList.push(localTrackStats[localVideoTrackID]); + } + + const subscribeStatsReport = await this.sdk.getWebrtcInternals()?.getSubscribePeerConnection()?.getStats(); + subscribeStatsReport?.forEach(stat => { + if (stat.type === 'inbound-rtp') { + const list = stat.kind === 'audio' ? this.remoteAudioTrackStatsList : this.remoteVideoTrackStatsList; + const bitrate = computeBitrate('bytesReceived', stat, getLastElement(list)); + list.push({ ...stat, bitrate }); + } + }); + } + + // eslint-disable-next-line complexity + buildReport(): DiagnosticsRTCStatsReport { + const lastPublishStats = getLastElement(this.peerStatsList)?.publish; + const lastSubscribeStats = getLastElement(this.peerStatsList)?.subscribe; + const publishRoundTripTime = lastPublishStats?.responsesReceived + ? (lastPublishStats?.totalRoundTripTime || 0) / lastPublishStats.responsesReceived + : 0; + const subscribeRoundTripTime = lastSubscribeStats?.responsesReceived + ? (lastSubscribeStats?.totalRoundTripTime || 0) / lastSubscribeStats.responsesReceived + : 0; + const roundTripTime = Number((((publishRoundTripTime + subscribeRoundTripTime) / 2) * 1000).toFixed(2)); + + const audioPacketsReceived = getLastElement(this.remoteAudioTrackStatsList)?.packetsReceived || 0; + const videoPacketsReceived = getLastElement(this.remoteVideoTrackStatsList)?.packetsReceived || 0; + + const ridAveragedAudioBitrateList = this.localAudioTrackStatsList.map(trackStatsMap => + trackStatsMap ? calculateAverage(Object.values(trackStatsMap), curr => curr.bitrate) : 0, + ); + + const ridAveragedVideoBitrateList = this.localVideoTrackStatsList.map(trackStatsMap => + trackStatsMap ? calculateAverage(Object.values(trackStatsMap), curr => curr.bitrate) : 0, + ); + const audioJitter = getLastElement(this.remoteAudioTrackStatsList)?.jitter || 0; + const videoJitter = getLastElement(this.remoteVideoTrackStatsList)?.jitter || 0; + const jitter = Math.max(audioJitter, videoJitter); + const lastLocalAudioTrackStats = getLastElement(this.localAudioTrackStatsList); + const lastLocalVideoTrackStats = getLastElement(this.localVideoTrackStatsList); + + return { + combined: { + roundTripTime, + packetsReceived: audioPacketsReceived + videoPacketsReceived, + packetsLost: lastSubscribeStats?.packetsLost || 0, + bytesSent: lastPublishStats?.bytesSent || 0, + bytesReceived: lastSubscribeStats?.bytesReceived || 0, + bitrateSent: calculateAverage(this.peerStatsList, curr => curr.publish?.bitrate), + bitrateReceived: calculateAverage(this.peerStatsList, curr => curr.subscribe?.bitrate), + jitter: jitter, + }, + audio: { + roundTripTime, + packetsReceived: audioPacketsReceived, + packetsLost: getLastElement(this.remoteAudioTrackStatsList)?.packetsLost || 0, + bytesReceived: getLastElement(this.remoteAudioTrackStatsList)?.bytesReceived || 0, + bitrateSent: calculateAverage(ridAveragedAudioBitrateList, curr => curr), + bitrateReceived: calculateAverage(this.remoteAudioTrackStatsList, curr => curr.bitrate), + bytesSent: lastLocalAudioTrackStats + ? Object.values(lastLocalAudioTrackStats).reduce((acc, curr) => acc + (curr.bytesSent || 0), 0) + : 0, + jitter: audioJitter, + }, + video: { + roundTripTime, + packetsLost: getLastElement(this.remoteVideoTrackStatsList)?.packetsLost || 0, + bytesReceived: getLastElement(this.remoteVideoTrackStatsList)?.bytesReceived || 0, + packetsReceived: videoPacketsReceived, + bitrateSent: calculateAverage(ridAveragedVideoBitrateList, curr => curr), + bitrateReceived: calculateAverage(this.remoteVideoTrackStatsList, curr => curr.bitrate), + bytesSent: lastLocalVideoTrackStats + ? Object.values(lastLocalVideoTrackStats).reduce((acc, curr) => acc + (curr.bytesSent || 0), 0) + : 0, + jitter: videoJitter, + }, + }; + } +} diff --git a/packages/hms-video-store/src/diagnostics/HMSDiagnostics.ts b/packages/hms-video-store/src/diagnostics/HMSDiagnostics.ts new file mode 100644 index 0000000000..3af667bd98 --- /dev/null +++ b/packages/hms-video-store/src/diagnostics/HMSDiagnostics.ts @@ -0,0 +1,248 @@ +import { ConnectivityCheck } from './ConnectivityCheck'; +import { DEFAULT_TEST_AUDIO_URL, diagnosticsRole, MIC_CHECK_RECORD_DURATION } from './constants'; +import { + ConnectivityCheckResult, + ConnectivityState, + HMSDiagnosticsInterface, + MediaPermissionCheck, +} from './interfaces'; +import { ErrorFactory } from '../error/ErrorFactory'; +import { HMSAction } from '../error/HMSAction'; +import { BuildGetMediaError } from '../error/utils'; +import { + HMSException, + HMSLocalAudioTrack, + HMSLocalVideoTrack, + HMSPeerType, + HMSPeerUpdate, + HMSRoomUpdate, + HMSUpdateListener, +} from '../internal'; +import { HMSAudioTrackSettingsBuilder, HMSTrackSettingsBuilder, HMSVideoTrackSettingsBuilder } from '../media/settings'; +import { HMSSdk } from '../sdk'; +import HMSRoom from '../sdk/models/HMSRoom'; +import { HMSLocalPeer } from '../sdk/models/peer'; +import { fetchWithRetry } from '../utils/fetch'; +import { validateMediaDevicesExistence, validateRTCPeerConnection } from '../utils/validations'; + +export class Diagnostics implements HMSDiagnosticsInterface { + private recordedAudio?: string = DEFAULT_TEST_AUDIO_URL; + private mediaRecorder?: MediaRecorder; + private connectivityCheck?: ConnectivityCheck; + private onStopMicCheck?: () => void; + + constructor(private sdk: HMSSdk, private sdkListener: HMSUpdateListener) { + this.sdk.setIsDiagnostics(true); + this.initSdkWithLocalPeer(); + } + + get localPeer() { + return this.sdk?.store.getLocalPeer(); + } + + checkBrowserSupport(): void { + validateMediaDevicesExistence(); + validateRTCPeerConnection(); + } + + async requestPermission(check: MediaPermissionCheck): Promise { + try { + const stream = await navigator.mediaDevices.getUserMedia(check); + stream.getTracks().forEach(track => track.stop()); + await this.sdk.deviceManager.init(true); + return { + audio: stream.getAudioTracks().length > 0, + video: stream.getVideoTracks().length > 0, + }; + } catch (err) { + throw BuildGetMediaError(err as Error, this.sdk.localTrackManager.getErrorType(!!check.video, !!check.audio)); + } + } + + async startCameraCheck(inputDevice?: string) { + this.initSdkWithLocalPeer(); + if (!this.localPeer) { + throw new Error('Local peer not found'); + } + + this.sdk.store.setSimulcastEnabled(false); + + this.localPeer.role = { + ...diagnosticsRole, + publishParams: { ...diagnosticsRole.publishParams, allowed: ['video'] }, + }; + const settings = new HMSTrackSettingsBuilder() + .video(new HMSVideoTrackSettingsBuilder().deviceId(inputDevice || 'default').build()) + .build(); + const tracks = await this.sdk?.localTrackManager.getLocalTracks({ audio: false, video: true }, settings); + const track = tracks?.find(track => track.type === 'video') as HMSLocalVideoTrack; + + if (!track) { + throw new Error('No video track found'); + } + + this.sdk?.deviceManager.init(true); + this.localPeer.videoTrack = track; + this.sdk?.listener?.onPeerUpdate(HMSPeerUpdate.PEER_LIST, [this.localPeer]); + } + + stopCameraCheck(): void { + this.localPeer?.videoTrack?.cleanup(); + if (this.localPeer) { + this.localPeer.videoTrack = undefined; + } + } + + async startMicCheck({ + inputDevice, + onError, + onStop, + time = MIC_CHECK_RECORD_DURATION, + }: { + inputDevice?: string; + onError?: (error: Error) => void; + onStop?: () => void; + time?: number; + }) { + this.initSdkWithLocalPeer((err: Error) => { + this.stopMicCheck(); + onError?.(err); + }); + const track = await this.getLocalAudioTrack(inputDevice); + this.sdk?.deviceManager.init(true); + if (!this.localPeer) { + throw new Error('Local peer not found'); + } + if (!track) { + throw new Error('No audio track found'); + } + + this.localPeer.audioTrack = track; + this.sdk?.initPreviewTrackAudioLevelMonitor(); + this.sdk?.listener?.onPeerUpdate(HMSPeerUpdate.PEER_LIST, [this.localPeer]); + + this.mediaRecorder = new MediaRecorder(track.stream.nativeStream); + const chunks: Blob[] = []; + + this.mediaRecorder.ondataavailable = function (e) { + chunks.push(e.data); + }; + + this.mediaRecorder.onstop = () => { + const blob = new Blob(chunks, { type: this.mediaRecorder?.mimeType }); + this.recordedAudio = URL.createObjectURL(blob); + this.onStopMicCheck?.(); + }; + + this.mediaRecorder.start(); + + const timeoutId = setTimeout(() => { + this.stopMicCheck(); + }, time); + + this.onStopMicCheck = () => { + clearTimeout(timeoutId); + onStop?.(); + }; + } + + stopMicCheck(): void { + this.mediaRecorder?.stop(); + this.localPeer?.audioTrack?.cleanup(); + if (this.localPeer) { + this.localPeer.audioTrack = undefined; + } + } + + getRecordedAudio() { + return this.recordedAudio; + } + + async startConnectivityCheck( + progress: (state: ConnectivityState) => void, + completed: (result: ConnectivityCheckResult) => void, + region?: string, + ) { + if (!this.sdk) { + throw new Error('SDK not found'); + } + + this.connectivityCheck = new ConnectivityCheck(this.sdk, this.sdkListener, progress, completed); + + const authToken = await this.getAuthToken(region); + await this.sdk.leave(); + await this.sdk.join({ authToken, userName: 'diagnostics-test' }, this.connectivityCheck); + this.sdk.addConnectionQualityListener({ + onConnectionQualityUpdate: qualityUpdates => { + this.connectivityCheck?.handleConnectionQualityUpdate(qualityUpdates); + }, + }); + } + + async stopConnectivityCheck(): Promise { + return this.connectivityCheck?.cleanupAndReport(); + } + + private initSdkWithLocalPeer(onError?: (error: Error) => void) { + this.sdkListener && + this.sdk?.initStoreAndManagers({ + ...this.sdkListener, + onError: (error: HMSException) => { + onError?.(error); + this.sdkListener.onError(error); + }, + }); + const localPeer = new HMSLocalPeer({ + name: 'diagnostics-peer', + role: diagnosticsRole, + type: HMSPeerType.REGULAR, + }); + + this.sdk?.store.addPeer(localPeer); + + const room = new HMSRoom('diagnostics-room'); + this.sdk.store.setRoom(room); + this.sdkListener.onRoomUpdate(HMSRoomUpdate.ROOM_PEER_COUNT_UPDATED, room); + this.sdk?.deviceManager.init(true); + } + + private async getAuthToken(region?: string): Promise { + const tokenAPIURL = new URL('https://api.100ms.live/v2/diagnostics/token'); + if (region) { + tokenAPIURL.searchParams.append('region', region); + } + const response = await fetchWithRetry( + tokenAPIURL.toString(), + { method: 'GET' }, + [429, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511], + ); + + const data = await response.json(); + + if (!response.ok) { + throw ErrorFactory.APIErrors.ServerErrors(data.code, HMSAction.GET_TOKEN, data.message, false); + } + + const { token } = data; + if (!token) { + throw Error(data.message); + } + return token; + } + + private async getLocalAudioTrack(inputDevice?: string) { + if (!this.localPeer) { + return; + } + + this.localPeer.role = { + ...diagnosticsRole, + publishParams: { ...diagnosticsRole.publishParams, allowed: ['audio'] }, + }; + const settings = new HMSTrackSettingsBuilder() + .audio(new HMSAudioTrackSettingsBuilder().deviceId(inputDevice || 'default').build()) + .build(); + const tracks = await this.sdk?.localTrackManager.getLocalTracks({ audio: true, video: false }, settings); + return tracks?.find(track => track.type === 'audio') as HMSLocalAudioTrack; + } +} diff --git a/packages/hms-video-store/src/diagnostics/constants.ts b/packages/hms-video-store/src/diagnostics/constants.ts new file mode 100644 index 0000000000..f1d0f12be6 --- /dev/null +++ b/packages/hms-video-store/src/diagnostics/constants.ts @@ -0,0 +1,47 @@ +import { HMSRole } from '../internal'; + +export const DIAGNOSTICS_ROOM_ID = '6669151f480e55c215299a4a'; + +export const CONNECTIVITY_TEST_DURATION = 20_000; +export const MIC_CHECK_RECORD_DURATION = 10_000; + +export const diagnosticsRole: HMSRole = { + name: 'diagnostics-role', + priority: 1, + publishParams: { + allowed: ['audio', 'video'], + audio: { bitRate: 32, codec: 'opus' }, + video: { + bitRate: 100, + codec: 'vp8', + frameRate: 30, + height: 720, + width: 1280, + }, + screen: { + bitRate: 100, + codec: 'vp8', + frameRate: 10, + height: 1080, + width: 1920, + }, + }, + subscribeParams: { + subscribeToRoles: [], + maxSubsBitRate: 3200, + }, + permissions: { + browserRecording: false, + changeRole: false, + endRoom: false, + hlsStreaming: false, + mute: false, + pollRead: false, + pollWrite: false, + removeOthers: false, + rtmpStreaming: false, + unmute: false, + }, +}; + +export const DEFAULT_TEST_AUDIO_URL = 'https://100ms.live/test-audio.wav'; diff --git a/packages/hms-video-store/src/diagnostics/index.ts b/packages/hms-video-store/src/diagnostics/index.ts new file mode 100644 index 0000000000..7985af83c6 --- /dev/null +++ b/packages/hms-video-store/src/diagnostics/index.ts @@ -0,0 +1,2 @@ +export { Diagnostics } from './HMSDiagnostics'; +export * from './interfaces'; diff --git a/packages/hms-video-store/src/diagnostics/interfaces.ts b/packages/hms-video-store/src/diagnostics/interfaces.ts new file mode 100644 index 0000000000..8be0202847 --- /dev/null +++ b/packages/hms-video-store/src/diagnostics/interfaces.ts @@ -0,0 +1,92 @@ +import { RTCIceCandidatePair } from '../connection/IConnectionObserver'; +import { HMSException, HMSTrack, HMSUpdateListener } from '../internal'; + +export enum ConnectivityState { + STARTING, + INIT_FETCHED, + SIGNAL_CONNECTED, + ICE_ESTABLISHED, + MEDIA_CAPTURED, + MEDIA_PUBLISHED, + COMPLETED, +} + +export interface HMSDiagnosticsConnectivityListener extends HMSUpdateListener { + onInitSuccess(websocketURL: string): void; + onSignallingSuccess(): void; + onICESuccess(isPublish: boolean): void; + onMediaPublished(track: HMSTrack): void; + onICECandidate(candidate: RTCIceCandidate, isPublish: boolean): void; + onSelectedICECandidatePairChange(candidatePair: RTCIceCandidatePair, isPublish: boolean): void; +} + +export interface MediaPermissionCheck { + audio?: boolean; + video?: boolean; +} + +export interface HMSDiagnosticsInterface { + requestPermission(check: MediaPermissionCheck): Promise; + checkBrowserSupport(): void; + startMicCheck(args: { + inputDevice?: string; + onError?: (error: Error) => void; + onStop?: () => void; + time?: number; + }): Promise; + getRecordedAudio(): string | undefined; + stopMicCheck(): void; + + startCameraCheck(inputDevice?: string): Promise; + stopCameraCheck(): void; + + startConnectivityCheck( + progress: (state: ConnectivityState) => void, + completed: (result: ConnectivityCheckResult) => void, + region?: string, + ): Promise; + stopConnectivityCheck(): Promise; +} + +export interface ConnectivityCheckResult { + testTimestamp: number; // System time in millis (UTC) + connectivityState?: ConnectivityState; // This is the initial state + signallingReport?: SignallingReport; + mediaServerReport?: MediaServerReport; + // deviceTestReport?: DeviceTestReport; + errors?: Array; +} + +export interface SignallingReport { + isConnected: boolean; // true if websocket was connected successfully + isInitConnected: boolean; // True if init call was successful + websocketUrl?: string; // websocket url +} + +export interface MediaServerReport { + stats?: DiagnosticsRTCStatsReport; // represents the overall stats of the call + isPublishICEConnected: boolean; // True if ICE connected successfully for both publish and subscribe + isSubscribeICEConnected: boolean; + connectionQualityScore?: number; // Average of all the downlink scores for this call for this peer + publishIceCandidatesGathered?: Array; // Publish ICE candidates + subscribeIceCandidatesGathered: Array; // Subscribe ICE candidates + publishICECandidatePairSelected?: RTCIceCandidatePair; // publish ICE pair + subscribeICECandidatePairSelected?: RTCIceCandidatePair; // subscribe ICE pair +} + +export interface DiagnosticsRTCStatsReport { + combined: DiagnosticsRTCStats; + audio: DiagnosticsRTCStats; + video: DiagnosticsRTCStats; +} + +export interface DiagnosticsRTCStats { + bytesSent: number; + bytesReceived: number; + packetsReceived: number; + packetsLost: number; + bitrateSent: number; + bitrateReceived: number; + roundTripTime: number; + jitter: number; +} diff --git a/packages/hms-video-store/src/end-call-feedback/feedback.ts b/packages/hms-video-store/src/end-call-feedback/feedback.ts new file mode 100644 index 0000000000..ea12075b51 --- /dev/null +++ b/packages/hms-video-store/src/end-call-feedback/feedback.ts @@ -0,0 +1,62 @@ +import { HMSSessionFeedback, HMSSessionInfo } from './interface'; +import { ErrorFactory } from '../error/ErrorFactory'; +import { HMSAction } from '../error/HMSAction'; +import { HMSException } from '../internal'; +import HMSLogger from '../utils/logger'; + +export class FeedbackService { + private static TAG = '[FeedbackService]'; + private static handleError(response: Response) { + if (response.status === 404) { + throw ErrorFactory.APIErrors.EndpointUnreachable(HMSAction.FEEDBACK, response.statusText); + } + if (response.status >= 400) { + throw ErrorFactory.APIErrors.ServerErrors(response.status, HMSAction.FEEDBACK, response?.statusText); + } + return; + } + + static async sendFeedback({ + token, + eventEndpoint = 'https://event.100ms.live', + info, + feedback, + }: { + token: string; + eventEndpoint?: string; + info: HMSSessionInfo; + feedback?: HMSSessionFeedback; + }): Promise { + HMSLogger.d( + this.TAG, + `sendFeedback: feedbackEndpoint=${eventEndpoint} peerId=${info.peer.peer_id} session=${info.peer.session_id} `, + ); + const url = new URL('v2/client/feedback', eventEndpoint); + const body = { + ...info, + payload: feedback, + }; + try { + const response = await fetch(url, { + headers: { Authorization: `Bearer ${token}` }, + body: JSON.stringify(body), + method: 'POST', + }); + try { + this.handleError(response); + return; + } catch (err) { + HMSLogger.e(this.TAG, 'error', (err as Error).message, response.status); + throw err instanceof HMSException + ? err + : ErrorFactory.APIErrors.ServerErrors(response.status, HMSAction.FEEDBACK, (err as Error).message); + } + } catch (err) { + const error = err as Error; + if (['Failed to fetch', 'NetworkError', 'ECONNRESET'].some(message => error.message.includes(message))) { + throw ErrorFactory.APIErrors.EndpointUnreachable(HMSAction.FEEDBACK, error.message); + } + throw error; + } + } +} diff --git a/packages/hms-video-store/src/end-call-feedback/index.ts b/packages/hms-video-store/src/end-call-feedback/index.ts new file mode 100644 index 0000000000..21f3f2147c --- /dev/null +++ b/packages/hms-video-store/src/end-call-feedback/index.ts @@ -0,0 +1,2 @@ +export * from './interface'; +export * from './feedback'; diff --git a/packages/hms-video-store/src/end-call-feedback/interface.ts b/packages/hms-video-store/src/end-call-feedback/interface.ts new file mode 100644 index 0000000000..69b8efbdec --- /dev/null +++ b/packages/hms-video-store/src/end-call-feedback/interface.ts @@ -0,0 +1,42 @@ +export interface HMSSessionFeedback { + // The question asked in the feedback. + question?: string; + // The rating given for the Session experience. + rating: number; + // The minimum rating allowed. + min_rating?: number; + // The maximum rating allowed. + max_rating?: number; + // Reasons for the given rating. + reasons?: string[]; + // Additional comments provided by the user. + comment?: string; +} + +export interface HMSSessionInfo { + peer: HMSSessionPeerInfo; + agent?: string; + device_id: string; + cluster: HMSSessionCluster; + timestamp: number; +} + +// Data structure for peer data in a session. +export interface HMSSessionPeerInfo { + peer_id: string; + role?: string; + joined_at?: number; + left_at?: number; + room_name?: string; + session_started_at?: number; + user_data?: string; + user_name?: string; + template_id?: string; + session_id?: string; + token?: string; +} + +// Data structure for session cluster. +export interface HMSSessionCluster { + websocket_url: string; +} diff --git a/packages/hms-video-store/src/error/ErrorCodes.ts b/packages/hms-video-store/src/error/ErrorCodes.ts index 24e00e6419..dccd5b6286 100644 --- a/packages/hms-video-store/src/error/ErrorCodes.ts +++ b/packages/hms-video-store/src/error/ErrorCodes.ts @@ -76,6 +76,9 @@ export const ErrorCodes = { // Selected device not detected on change SELECTED_DEVICE_MISSING: 3014, + + // Track is publishing with no data, can happen when a whatsapp call is ongoing before 100ms call in mweb + NO_DATA: 3015, }, WebrtcErrors: { diff --git a/packages/hms-video-store/src/error/ErrorFactory.ts b/packages/hms-video-store/src/error/ErrorFactory.ts index 3ee246f9f7..c4b258c4a8 100644 --- a/packages/hms-video-store/src/error/ErrorFactory.ts +++ b/packages/hms-video-store/src/error/ErrorFactory.ts @@ -8,6 +8,8 @@ import { ErrorCodes } from './ErrorCodes'; import { HMSAction } from './HMSAction'; import { HMSException } from './HMSException'; +import { HMSTrackException } from './HMSTrackException'; +import { HMSTrackExceptionTrackType } from '../media/tracks/HMSTrackExceptionTrackType'; import { HMSSignalMethod } from '../signal/jsonrpc/models'; const terminalActions: (HMSSignalMethod | HMSAction)[] = [ @@ -36,7 +38,7 @@ export const ErrorFactory = { ErrorCodes.WebSocketConnectionErrors.WEBSOCKET_CONNECTION_LOST, 'WebSocketConnectionLost', action, - `Network connection lost `, + `Network connection lost`, description, ); }, @@ -98,52 +100,56 @@ export const ErrorFactory = { TracksErrors: { GenericTrack(action: HMSAction, description = '') { - return new HMSException( + return new HMSTrackException( ErrorCodes.TracksErrors.GENERIC_TRACK, 'GenericTrack', action, `[TRACK]: ${description}`, `[TRACK]: ${description}`, + HMSTrackExceptionTrackType.AUDIO_VIDEO, ); }, - CantAccessCaptureDevice(action: HMSAction, deviceInfo: string, description = '') { - return new HMSException( + return new HMSTrackException( ErrorCodes.TracksErrors.CANT_ACCESS_CAPTURE_DEVICE, 'CantAccessCaptureDevice', action, `User denied permission to access capture device - ${deviceInfo}`, description, + deviceInfo as HMSTrackExceptionTrackType, ); }, DeviceNotAvailable(action: HMSAction, deviceInfo: string, description = '') { - return new HMSException( + return new HMSTrackException( ErrorCodes.TracksErrors.DEVICE_NOT_AVAILABLE, 'DeviceNotAvailable', action, `[TRACK]: Capture device is no longer available - ${deviceInfo}`, description, + deviceInfo as HMSTrackExceptionTrackType, ); }, DeviceInUse(action: HMSAction, deviceInfo: string, description = '') { - return new HMSException( + return new HMSTrackException( ErrorCodes.TracksErrors.DEVICE_IN_USE, 'DeviceInUse', action, `[TRACK]: Capture device is in use by another application - ${deviceInfo}`, description, + deviceInfo as HMSTrackExceptionTrackType, ); }, DeviceLostMidway(action: HMSAction, deviceInfo: string, description = '') { - return new HMSException( + return new HMSTrackException( ErrorCodes.TracksErrors.DEVICE_LOST_MIDWAY, 'DeviceLostMidway', action, `Lost access to capture device midway - ${deviceInfo}`, description, + deviceInfo as HMSTrackExceptionTrackType, ); }, @@ -152,103 +158,123 @@ export const ErrorFactory = { description = '', message = `There is no media to return. Please select either video or audio or both.`, ) { - return new HMSException( + return new HMSTrackException( ErrorCodes.TracksErrors.NOTHING_TO_RETURN, 'NothingToReturn', action, message, description, + HMSTrackExceptionTrackType.AUDIO_VIDEO, ); }, InvalidVideoSettings(action: HMSAction, description = '') { - return new HMSException( + return new HMSTrackException( ErrorCodes.TracksErrors.INVALID_VIDEO_SETTINGS, 'InvalidVideoSettings', action, `Cannot enable simulcast when no video settings are provided`, description, + HMSTrackExceptionTrackType.VIDEO, ); }, AutoplayBlocked(action: HMSAction, description = '') { - return new HMSException( + return new HMSTrackException( ErrorCodes.TracksErrors.AUTOPLAY_ERROR, 'AutoplayBlocked', action, "Autoplay blocked because the user didn't interact with the document first", description, + HMSTrackExceptionTrackType.AUDIO, ); }, CodecChangeNotPermitted(action: HMSAction, description = '') { - return new HMSException( + return new HMSTrackException( ErrorCodes.TracksErrors.CODEC_CHANGE_NOT_PERMITTED, 'CodecChangeNotPermitted', action, `Codec can't be changed mid call.`, description, + HMSTrackExceptionTrackType.AUDIO_VIDEO, ); }, OverConstrained(action: HMSAction, deviceInfo: string, description = '') { - return new HMSException( + return new HMSTrackException( ErrorCodes.TracksErrors.OVER_CONSTRAINED, 'OverConstrained', action, `[TRACK]: Requested constraints cannot be satisfied with the device hardware - ${deviceInfo}`, description, + deviceInfo as HMSTrackExceptionTrackType, ); }, NoAudioDetected(action: HMSAction, description = 'Please check the mic or use another audio input') { - return new HMSException( + return new HMSTrackException( ErrorCodes.TracksErrors.NO_AUDIO_DETECTED, 'NoAudioDetected', action, 'No audio input detected from microphone', description, + HMSTrackExceptionTrackType.AUDIO, ); }, SystemDeniedPermission(action: HMSAction, deviceInfo: string, description = '') { - return new HMSException( + return new HMSTrackException( ErrorCodes.TracksErrors.SYSTEM_DENIED_PERMISSION, 'SystemDeniedPermission', action, `Operating System denied permission to access capture device - ${deviceInfo}`, description, + deviceInfo as HMSTrackExceptionTrackType, ); }, CurrentTabNotShared() { - return new HMSException( + return new HMSTrackException( ErrorCodes.TracksErrors.CURRENT_TAB_NOT_SHARED, 'CurrentTabNotShared', HMSAction.TRACK, 'The app requires you to share the current tab', 'You must screen share the current tab in order to proceed', + HMSTrackExceptionTrackType.SCREEN, ); }, AudioPlaybackError(description: string) { - return new HMSException( + return new HMSTrackException( ErrorCodes.TracksErrors.AUDIO_PLAYBACK_ERROR, 'Audio playback error', HMSAction.TRACK, description, description, + HMSTrackExceptionTrackType.AUDIO, ); }, SelectedDeviceMissing(deviceType: string) { - return new HMSException( + return new HMSTrackException( ErrorCodes.TracksErrors.SELECTED_DEVICE_MISSING, 'SelectedDeviceMissing', HMSAction.TRACK, `Could not detect selected ${deviceType} device`, `Please check connection to the ${deviceType} device`, - false, + deviceType as HMSTrackExceptionTrackType, + ); + }, + + NoDataInTrack(description: string) { + return new HMSTrackException( + ErrorCodes.TracksErrors.NO_DATA, + 'Track does not have any data', + HMSAction.TRACK, + description, + 'This could possibily due to another application taking priority over the access to camera or microphone or due to an incoming call', + HMSTrackExceptionTrackType.AUDIO_VIDEO, ); }, }, diff --git a/packages/hms-video-store/src/error/HMSAction.ts b/packages/hms-video-store/src/error/HMSAction.ts index 52cbdfc6e4..b21b7279f5 100644 --- a/packages/hms-video-store/src/error/HMSAction.ts +++ b/packages/hms-video-store/src/error/HMSAction.ts @@ -16,4 +16,5 @@ export enum HMSAction { VALIDATION = 'VALIDATION', PLAYLIST = 'PLAYLIST', PREVIEW = 'PREVIEW', + FEEDBACK = 'FEEDBACK', } diff --git a/packages/hms-video-store/src/error/HMSException.ts b/packages/hms-video-store/src/error/HMSException.ts index 03c6afa687..88513d117d 100644 --- a/packages/hms-video-store/src/error/HMSException.ts +++ b/packages/hms-video-store/src/error/HMSException.ts @@ -38,13 +38,13 @@ export class HMSException extends Error implements IAnalyticsPropertiesProvider toString() { return `{ - code: ${this.code}; - name: ${this.name}; - action: ${this.action}; - message: ${this.message}; - description: ${this.description}; - isTerminal: ${this.isTerminal}; - nativeError: ${this.nativeError?.message}; - }`; + code: ${this.code}; + name: ${this.name}; + action: ${this.action}; + message: ${this.message}; + description: ${this.description}; + isTerminal: ${this.isTerminal}; + nativeError: ${this.nativeError?.message}; + }`; } } diff --git a/packages/hms-video-store/src/error/HMSTrackException.ts b/packages/hms-video-store/src/error/HMSTrackException.ts new file mode 100644 index 0000000000..1a8f9c666b --- /dev/null +++ b/packages/hms-video-store/src/error/HMSTrackException.ts @@ -0,0 +1,37 @@ +import { HMSAction } from './HMSAction'; +import { HMSException } from './HMSException'; +import { HMSTrackExceptionTrackType } from '../media/tracks/HMSTrackExceptionTrackType'; +import { HMSSignalMethod } from '../signal/jsonrpc/models'; + +export class HMSTrackException extends HMSException { + constructor( + public readonly code: number, + public name: string, + action: HMSAction | HMSSignalMethod, + public message: string, + public description: string, + public trackType: HMSTrackExceptionTrackType, + ) { + super(code, name, action, message, description, false); + } + + toAnalyticsProperties() { + return { + ...super.toAnalyticsProperties(), + track_type: this.trackType, + }; + } + + toString() { + return `{ + code: ${this.code}; + name: ${this.name}; + action: ${this.action}; + message: ${this.message}; + description: ${this.description}; + isTerminal: ${this.isTerminal}; + nativeError: ${this.nativeError?.message}; + trackType: ${this.trackType}; + }`; + } +} diff --git a/packages/hms-video-store/src/error/utils.ts b/packages/hms-video-store/src/error/utils.ts index 42dea262ce..36c5efbbbf 100644 --- a/packages/hms-video-store/src/error/utils.ts +++ b/packages/hms-video-store/src/error/utils.ts @@ -1,7 +1,7 @@ import adapter from 'webrtc-adapter'; import { ErrorFactory } from './ErrorFactory'; import { HMSAction } from './HMSAction'; -import { HMSException } from './HMSException'; +import { HMSTrackException } from './HMSTrackException'; export enum HMSGetMediaActions { UNKNOWN = 'unknown(video or audio)', @@ -13,13 +13,15 @@ export enum HMSGetMediaActions { function getDefaultError(error: string, deviceInfo: string) { const message = error.toLowerCase(); + let exception = ErrorFactory.TracksErrors.GenericTrack(HMSAction.TRACK, error); + if (message.includes('device not found')) { - return ErrorFactory.TracksErrors.DeviceNotAvailable(HMSAction.TRACK, deviceInfo, error); + exception = ErrorFactory.TracksErrors.DeviceNotAvailable(HMSAction.TRACK, deviceInfo, error); } else if (message.includes('permission denied')) { - return ErrorFactory.TracksErrors.CantAccessCaptureDevice(HMSAction.TRACK, deviceInfo, error); - } else { - return ErrorFactory.TracksErrors.GenericTrack(HMSAction.TRACK, error); + exception = ErrorFactory.TracksErrors.CantAccessCaptureDevice(HMSAction.TRACK, deviceInfo, error); } + + return exception; } /** @@ -31,7 +33,7 @@ function getDefaultError(error: string, deviceInfo: string) { * System blocked - NotAllowedError - Permission denied by system */ // eslint-disable-next-line complexity -function convertMediaErrorToHMSException(err: Error, deviceInfo = ''): HMSException { +function convertMediaErrorToHMSException(err: Error, deviceInfo = ''): HMSTrackException { /** * Note: Adapter detects all chromium browsers as 'chrome' */ @@ -70,7 +72,7 @@ function convertMediaErrorToHMSException(err: Error, deviceInfo = ''): HMSExcept } } -export function BuildGetMediaError(err: Error, deviceInfo: string): HMSException { +export function BuildGetMediaError(err: Error, deviceInfo: string): HMSTrackException { const exception = convertMediaErrorToHMSException(err, deviceInfo); exception.addNativeError(err); return exception; diff --git a/packages/hms-video-store/src/events/EventBus.ts b/packages/hms-video-store/src/events/EventBus.ts index 193f26b9bd..096b4bba32 100644 --- a/packages/hms-video-store/src/events/EventBus.ts +++ b/packages/hms-video-store/src/events/EventBus.ts @@ -17,6 +17,10 @@ import { ITrackAudioLevelUpdate } from '../utils/track-audio-level-monitor'; export class EventBus { private eventEmitter: EventEmitter = new EventEmitter(); + readonly analytics: HMSInternalEvent = new HMSInternalEvent( + HMSEvents.ANALYTICS, + this.eventEmitter, + ); readonly deviceChange = new HMSInternalEvent(HMSEvents.DEVICE_CHANGE, this.eventEmitter); readonly localAudioEnabled = new HMSInternalEvent<{ enabled: boolean; track: HMSLocalAudioTrack }>( HMSEvents.LOCAL_AUDIO_ENABLED, @@ -26,6 +30,8 @@ export class EventBus { HMSEvents.LOCAL_VIDEO_ENABLED, this.eventEmitter, ); + readonly localVideoUnmutedNatively = new HMSInternalEvent(HMSEvents.LOCAL_VIDEO_UNMUTED_NATIVELY, this.eventEmitter); + readonly localAudioUnmutedNatively = new HMSInternalEvent(HMSEvents.LOCAL_AUDIO_UNMUTED_NATIVELY, this.eventEmitter); /** * Emitter which processes raw RTC stats from rtcStatsUpdate and calls client callback @@ -50,8 +56,6 @@ export class EventBus { this.eventEmitter, ); - readonly analytics = new HMSInternalEvent(HMSEvents.ANALYTICS, this.eventEmitter); - readonly policyChange = new HMSInternalEvent(HMSEvents.POLICY_CHANGE, this.eventEmitter); readonly localRoleUpdate = new HMSInternalEvent<{ oldRole: HMSRole; newRole: HMSRole }>( diff --git a/packages/hms-video-store/src/index.ts b/packages/hms-video-store/src/index.ts index 5fafe2bfd9..8d44ae20d5 100644 --- a/packages/hms-video-store/src/index.ts +++ b/packages/hms-video-store/src/index.ts @@ -11,6 +11,8 @@ export type { IHMSNotifications as HMSNotifications } from './schema/notificatio export * from './selectors'; export * from './webrtc-stats'; export { + HMSAudioMode, + HMSAudioDeviceCategory, HMSLogLevel, HMSAudioPluginType, HMSVideoPluginType, @@ -18,6 +20,8 @@ export { parsedUserAgent, simulcastMapping, DeviceType, + HMSPeerType, + getAudioDeviceCategory, } from './internal'; export type { @@ -51,10 +55,20 @@ export type { HMSPollQuestionOption, HMSQuizLeaderboardResponse, HMSQuizLeaderboardSummary, + HMSTranscriptionInfo, + HMSICEServer, } from './internal'; +export { EventBus } from './events/EventBus'; export { HMSReactiveStore } from './reactive-store/HMSReactiveStore'; -export { HMSPluginUnsupportedTypes, HMSRecordingState } from './internal'; +export { + HMSPluginUnsupportedTypes, + HMSRecordingState, + HLSPlaylistType, + HLSStreamType, + HMSTranscriptionMode, + HMSTranscriptionState, +} from './internal'; export type { HMSVideoPlugin, HMSAudioPlugin, @@ -62,3 +76,7 @@ export type { HMSPluginSupportResult, HMSFrameworkInfo, } from './internal'; +export * from './diagnostics'; +export { DomainCategory } from './analytics/AnalyticsEventDomains'; + +export { HMSTrackExceptionTrackType } from './media/tracks/HMSTrackExceptionTrackType'; diff --git a/packages/hms-video-store/src/interfaces/config.ts b/packages/hms-video-store/src/interfaces/config.ts index 6a2f33e370..23b88c4b58 100644 --- a/packages/hms-video-store/src/interfaces/config.ts +++ b/packages/hms-video-store/src/interfaces/config.ts @@ -1,10 +1,17 @@ import InitialSettings from './settings'; +export type HMSICEServer = { + urls: string[]; + userName?: string; + password?: string; +}; + /** * the config object tells the SDK options you want to join with * @link https://docs.100ms.live/javascript/v2/features/preview * @link https://docs.100ms.live/javascript/v2/features/join */ + export interface HMSConfig { /** * the name of the peer, can be later accessed via peer.name and can also be changed mid call. @@ -35,10 +42,6 @@ export interface HMSConfig { audioSinkElementId?: string; autoVideoSubscribe?: boolean; initEndpoint?: string; - /** - * Request Camera/Mic permissions irrespective of role to avoid delay in getting device list - */ - alwaysRequestPermissions?: boolean; /** * Enable to get a network quality score while in preview. The score ranges from -1 to 5. * -1 when we are not able to connect to 100ms servers within an expected time limit @@ -53,10 +56,14 @@ export interface HMSConfig { */ autoManageVideo?: boolean; /** - * if this flag is enabled, wake lock will be acquired automatically(if supported) when joining the room, so the device + * if this flag is enabled, wake lock will be acquired automatically (if supported) when joining the room, so the device * will be kept awake. */ autoManageWakeLock?: boolean; + /** + * use custom STUN/TURN servers for media connection (advanced) + */ + iceServers?: HMSICEServer[]; } export interface HMSMidCallPreviewConfig { diff --git a/packages/hms-video-store/src/interfaces/hms.ts b/packages/hms-video-store/src/interfaces/hms.ts index 3a1582db99..ead4c82e4c 100644 --- a/packages/hms-video-store/src/interfaces/hms.ts +++ b/packages/hms-video-store/src/interfaces/hms.ts @@ -8,13 +8,15 @@ import { HMSPlaylistManager, HMSPlaylistSettings } from './playlist'; import { HMSPreviewListener } from './preview-listener'; import { HMSRole } from './role'; import { HMSRoleChangeRequest } from './role-change-request'; -import { HMSHLS, HMSRecording, HMSRTMP } from './room'; +import { HMSHLS, HMSRecording, HMSRTMP, HMSTranscriptionInfo } from './room'; import { RTMPRecordingConfig } from './rtmp-recording-config'; import { HMSInteractivityCenter, HMSSessionStore } from './session-store'; import { HMSScreenShareConfig } from './track-settings'; +import { TranscriptionConfig } from './transcription-config'; import { HMSAudioListener, HMSConnectionQualityListener, HMSUpdateListener } from './update-listener'; import { HMSAnalyticsLevel } from '../analytics/AnalyticsEventLevel'; import { IAudioOutputManager } from '../device-manager/AudioOutputManager'; +import { HMSSessionFeedback } from '../end-call-feedback'; import { HMSRemoteTrack, HMSTrackSource } from '../media/tracks'; import { HMSWebrtcInternals } from '../rtc-stats/HMSWebrtcInternals'; import { HMSPeerListIterator } from '../sdk/HMSPeerListIterator'; @@ -61,9 +63,12 @@ export interface HMSInterface { */ startHLSStreaming(params?: HLSConfig): Promise; stopHLSStreaming(params?: HLSConfig): Promise; + startTranscription(params: TranscriptionConfig): Promise; + stopTranscription(params: TranscriptionConfig): Promise; getRecordingState(): HMSRecording | undefined; getRTMPState(): HMSRTMP | undefined; getHLSState(): HMSHLS | undefined; + getTranscriptionState(): HMSTranscriptionInfo[] | undefined; changeName(name: string): Promise; changeMetadata(metadata: string): Promise; @@ -100,4 +105,5 @@ export interface HMSInterface { getPeerListIterator(options?: HMSPeerListIteratorOptions): HMSPeerListIterator; updatePlaylistSettings(options: HMSPlaylistSettings): void; + submitSessionFeedback(feedback: HMSSessionFeedback, eventEndpoint?: string): Promise; } diff --git a/packages/hms-video-store/src/interfaces/index.ts b/packages/hms-video-store/src/interfaces/index.ts index 8bfb7576e5..f18940854f 100644 --- a/packages/hms-video-store/src/interfaces/index.ts +++ b/packages/hms-video-store/src/interfaces/index.ts @@ -19,3 +19,4 @@ export * from './webrtc-stats'; export * from './framework-info'; export * from './get-token'; export * from './session-store'; +export * from './transcription-config'; diff --git a/packages/hms-video-store/src/interfaces/peer/hms-peer.ts b/packages/hms-video-store/src/interfaces/peer/hms-peer.ts index 1e1b557f54..e3100831ad 100644 --- a/packages/hms-video-store/src/interfaces/peer/hms-peer.ts +++ b/packages/hms-video-store/src/interfaces/peer/hms-peer.ts @@ -1,6 +1,11 @@ import { HMSAudioTrack, HMSTrack, HMSVideoTrack } from '../../media/tracks'; import { HMSRole } from '../role'; +export enum HMSPeerType { + SIP = 'sip', + REGULAR = 'regular', +} + export interface HMSPeer { peerId: string; name: string; @@ -16,6 +21,7 @@ export interface HMSPeer { groups?: string[]; realtime?: boolean; isHandRaised: boolean; + type: HMSPeerType; updateRole(newRole: HMSRole): void; updateName(newName: string): void; diff --git a/packages/hms-video-store/src/interfaces/peer/index.ts b/packages/hms-video-store/src/interfaces/peer/index.ts index 861ab7619d..c92ba38909 100644 --- a/packages/hms-video-store/src/interfaces/peer/index.ts +++ b/packages/hms-video-store/src/interfaces/peer/index.ts @@ -2,3 +2,4 @@ export type { HMSPeer } from './hms-peer'; export type { HMSLocalPeer } from './hms-local-peer'; export type { HMSRemotePeer } from './hms-remote-peer'; export type { HMSConnectionQuality } from './connection-quality'; +export { HMSPeerType } from './hms-peer'; diff --git a/packages/hms-video-store/src/interfaces/preview-listener.ts b/packages/hms-video-store/src/interfaces/preview-listener.ts index 56ac6019ff..4c88d21b2a 100644 --- a/packages/hms-video-store/src/interfaces/preview-listener.ts +++ b/packages/hms-video-store/src/interfaces/preview-listener.ts @@ -1,7 +1,7 @@ import { DeviceChangeListener } from './devices'; import { HMSPeer } from './peer'; import { HMSRoom } from './room'; -import { HMSPeerUpdate, HMSRoomUpdate } from './update-listener'; +import { HMSPeerUpdate, HMSRoomUpdate, HMSTrackUpdate } from './update-listener'; import { HMSException } from '../error/HMSException'; import { HMSTrack } from '../media/tracks/HMSTrack'; @@ -13,4 +13,6 @@ export interface HMSPreviewListener extends DeviceChangeListener { onRoomUpdate(type: HMSRoomUpdate, room: HMSRoom): void; onPeerUpdate(type: HMSPeerUpdate, peer: HMSPeer | HMSPeer[] | null): void; onNetworkQuality?(score: number): void; + // This is needed to mute audio when there is an error in device change + onTrackUpdate(type: HMSTrackUpdate, track: HMSTrack, peer: HMSPeer): void; } diff --git a/packages/hms-video-store/src/interfaces/role.ts b/packages/hms-video-store/src/interfaces/role.ts index c20ad3ffe7..75d414c807 100644 --- a/packages/hms-video-store/src/interfaces/role.ts +++ b/packages/hms-video-store/src/interfaces/role.ts @@ -1,3 +1,4 @@ +import { HMSTranscriptionMode } from './room'; import { SimulcastLayers } from './simulcast-layers'; export type HMSRoleName = string; @@ -23,6 +24,7 @@ export interface HMSRole { pollRead: boolean; pollWrite: boolean; whiteboard?: Array; + transcriptions?: Record>; }; priority: number; } diff --git a/packages/hms-video-store/src/interfaces/room.ts b/packages/hms-video-store/src/interfaces/room.ts index 21dc36e5ef..073851b2f0 100644 --- a/packages/hms-video-store/src/interfaces/room.ts +++ b/packages/hms-video-store/src/interfaces/room.ts @@ -33,14 +33,13 @@ export interface HMSRoom { description?: string; max_size?: number; large_room_optimization?: boolean; - /** - * @alpha - */ isEffectsEnabled?: boolean; - /** - * @alpha - */ + disableNoneLayerRequest?: boolean; + isVBEnabled?: boolean; effectsKey?: string; + isHipaaEnabled?: boolean; + isNoiseCancellationEnabled?: boolean; + transcriptions?: HMSTranscriptionInfo[]; } export interface HMSRecording { @@ -95,11 +94,45 @@ export interface HMSHLS { error?: HMSException; } +export enum HLSPlaylistType { + DVR = 'dvr', + NO_DVR = 'no-dvr', +} + +export enum HLSStreamType { + REGULAR = 'regular', + SCREEN = 'screen', + COMPOSITE = 'composite', +} export interface HLSVariant { url: string; + playlist_type?: HLSPlaylistType; meetingURL?: string; metadata?: string; startedAt?: Date; initialisedAt?: Date; state?: HMSStreamingState; + stream_type?: HLSStreamType; +} + +/* +Transcription related details +*/ +export enum HMSTranscriptionState { + INITIALISED = 'initialised', + STARTED = 'started', + STOPPED = 'stopped', + FAILED = 'failed', +} +export enum HMSTranscriptionMode { + CAPTION = 'caption', +} +export interface HMSTranscriptionInfo { + state?: HMSTranscriptionState; + mode?: HMSTranscriptionMode; + initialised_at?: Date; + started_at?: Date; + updated_at?: Date; + stopped_at?: Date; + error?: HMSException; } diff --git a/packages/hms-video-store/src/interfaces/session-store/interactivity-center.ts b/packages/hms-video-store/src/interfaces/session-store/interactivity-center.ts index 9a46fc7868..a5d413f8f4 100644 --- a/packages/hms-video-store/src/interfaces/session-store/interactivity-center.ts +++ b/packages/hms-video-store/src/interfaces/session-store/interactivity-center.ts @@ -20,6 +20,7 @@ export interface HMSInteractivityCenter { stopPoll(pollID: string): Promise; addResponsesToPoll(pollID: string, responses: HMSPollQuestionResponseCreateParams[]): Promise; fetchLeaderboard(pollID: string, offset: number, count: number): Promise; + getPollResponses(poll: HMSPoll, self: boolean): Promise; getPolls(): Promise; /** @alpha */ whiteboard: HMSWhiteboardInteractivityCenter; diff --git a/packages/hms-video-store/src/interfaces/session-store/whiteboard.ts b/packages/hms-video-store/src/interfaces/session-store/whiteboard.ts index e8104ea554..bd16b13b9d 100644 --- a/packages/hms-video-store/src/interfaces/session-store/whiteboard.ts +++ b/packages/hms-video-store/src/interfaces/session-store/whiteboard.ts @@ -15,9 +15,21 @@ export interface HMSWhiteboard { open?: boolean; // whether whiteboard is open or not title?: string; owner?: string; // user id for whiteboard owner - addr?: string; // address to be used to connect to whiteboard service - token?: string; // security token to be used for whiteboard API permissions?: Array; presence?: boolean; attributes?: Array<{ name: string; value: unknown }>; + /** + * the URL that needs to be used as the iframe src + */ + url?: string; + /** + * address to be used to connect to whiteboard grpc communication service + * @internal + */ + addr?: string; + /** + * security token to be used for whiteboard API + * @internal + */ + token?: string; } diff --git a/packages/hms-video-store/src/interfaces/settings.ts b/packages/hms-video-store/src/interfaces/settings.ts index 3c7b4434e0..fa3cfb61fd 100644 --- a/packages/hms-video-store/src/interfaces/settings.ts +++ b/packages/hms-video-store/src/interfaces/settings.ts @@ -1,9 +1,12 @@ +import { HMSAudioMode } from './track-settings'; + export default interface InitialSettings { isAudioMuted?: boolean; isVideoMuted?: boolean; audioInputDeviceId?: string; audioOutputDeviceId?: string; videoDeviceId?: string; + audioMode?: HMSAudioMode; /** * When a peer joins the room for the first time or when a device change happens, * after selecting the mic for audio input, we try to find the matching output device diff --git a/packages/hms-video-store/src/interfaces/track-settings.ts b/packages/hms-video-store/src/interfaces/track-settings.ts index bb9ecc395f..c8a402f158 100644 --- a/packages/hms-video-store/src/interfaces/track-settings.ts +++ b/packages/hms-video-store/src/interfaces/track-settings.ts @@ -19,12 +19,18 @@ export enum HMSFacingMode { RIGHT = 'right', } +export enum HMSAudioMode { + VOICE = 'voice', + MUSIC = 'music', +} + export interface HMSAudioTrackSettings { volume?: number; codec?: HMSAudioCodec; maxBitrate?: number; deviceId?: string; advanced?: Array; + audioMode?: HMSAudioMode; } export interface HMSVideoTrackSettings { diff --git a/packages/hms-video-store/src/interfaces/transcription-config.ts b/packages/hms-video-store/src/interfaces/transcription-config.ts new file mode 100644 index 0000000000..9294bfc851 --- /dev/null +++ b/packages/hms-video-store/src/interfaces/transcription-config.ts @@ -0,0 +1,5 @@ +import { HMSTranscriptionMode } from './room'; + +export interface TranscriptionConfig { + mode: HMSTranscriptionMode; +} diff --git a/packages/hms-video-store/src/interfaces/update-listener.ts b/packages/hms-video-store/src/interfaces/update-listener.ts index c870e29fec..adbe9973ad 100644 --- a/packages/hms-video-store/src/interfaces/update-listener.ts +++ b/packages/hms-video-store/src/interfaces/update-listener.ts @@ -17,6 +17,7 @@ export enum HMSRoomUpdate { SERVER_RECORDING_STATE_UPDATED = 'SERVER_RECORDING_STATE_UPDATED', RTMP_STREAMING_STATE_UPDATED = 'RTMP_STREAMING_STATE_UPDATED', HLS_STREAMING_STATE_UPDATED = 'HLS_STREAMING_STATE_UPDATED', + TRANSCRIPTION_STATE_UPDATED = 'TRANSCRIPTION_STATE_UPDATED', ROOM_PEER_COUNT_UPDATED = 'ROOM_PEER_COUNT_UPDATED', } @@ -54,7 +55,6 @@ export enum HMSPollsUpdate { POLL_STOPPED, POLLS_LIST, POLL_STATS_UPDATED, - // POLL_LEADERBOARD_SHARED, } export interface HMSAudioListener { @@ -83,6 +83,7 @@ export interface HMSUpdateListener extends DeviceChangeListener, SessionStoreLis onError(error: HMSException): void; onReconnecting(error: HMSException): void; onReconnected(): void; + onSFUMigration?: () => void; onRoleChangeRequest(request: HMSRoleChangeRequest): void; onRoleUpdate(newRole: string): void; onChangeTrackStateRequest(request: HMSChangeTrackStateRequest): void; diff --git a/packages/hms-video-store/src/interfaces/webrtc-stats.ts b/packages/hms-video-store/src/interfaces/webrtc-stats.ts index 44b8631c1e..05d5613e25 100644 --- a/packages/hms-video-store/src/interfaces/webrtc-stats.ts +++ b/packages/hms-video-store/src/interfaces/webrtc-stats.ts @@ -37,6 +37,7 @@ interface MissingOutboundStats extends RTCOutboundRtpStreamStats, MissingCommonS export interface MissingInboundStats extends RTCInboundRtpStreamStats, MissingCommonStats { bytesReceived?: number; + framesReceived?: number; framesDropped?: number; jitter?: number; packetsLost?: number; @@ -50,6 +51,13 @@ export interface MissingInboundStats extends RTCInboundRtpStreamStats, MissingCo fecPacketsDiscarded?: number; fecPacketsReceived?: number; totalSamplesDuration?: number; + pauseCount?: number; + totalPausesDuration?: number; + freezeCount?: number; + totalFreezesDuration?: number; + jitterBufferDelay?: number; + jitterBufferEmittedCount?: number; + estimatedPlayoutTimestamp?: DOMHighResTimeStamp; } export type PeerConnectionType = 'publish' | 'subscribe'; diff --git a/packages/hms-video-store/src/internal.ts b/packages/hms-video-store/src/internal.ts index 22ad33d6c9..dda7d2ba85 100644 --- a/packages/hms-video-store/src/internal.ts +++ b/packages/hms-video-store/src/internal.ts @@ -11,6 +11,7 @@ export * from './utils/media'; export * from './utils/device-error'; export * from './utils/support'; export * from './error/HMSException'; +export * from './error/HMSTrackException'; export * from './interfaces'; export * from './rtc-stats'; export * from './plugins'; diff --git a/packages/hms-video-store/src/media/settings/HMSAudioTrackSettings.ts b/packages/hms-video-store/src/media/settings/HMSAudioTrackSettings.ts index c22a5b42f0..20aa877ac0 100644 --- a/packages/hms-video-store/src/media/settings/HMSAudioTrackSettings.ts +++ b/packages/hms-video-store/src/media/settings/HMSAudioTrackSettings.ts @@ -1,26 +1,19 @@ +import { standardMediaConstraints } from './constants'; import { IAnalyticsPropertiesProvider } from '../../analytics/IAnalyticsPropertiesProvider'; -import { HMSAudioCodec, HMSAudioTrackSettings as IHMSAudioTrackSettings } from '../../interfaces'; +import { HMSAudioCodec, HMSAudioMode, HMSAudioTrackSettings as IHMSAudioTrackSettings } from '../../interfaces'; export class HMSAudioTrackSettingsBuilder { private _volume = 1.0; private _codec?: HMSAudioCodec = HMSAudioCodec.OPUS; private _maxBitrate?: number = 32; private _deviceId = 'default'; + private _audioMode: HMSAudioMode = HMSAudioMode.VOICE; private _advanced: Array = [ - // @ts-ignore - { googEchoCancellation: { exact: true } }, - // @ts-ignore - { googExperimentalEchoCancellation: { exact: true } }, - // @ts-ignore + ...standardMediaConstraints, { autoGainControl: { exact: true } }, // @ts-ignore { noiseSuppression: { exact: true } }, - // @ts-ignore - { googHighpassFilter: { exact: true } }, - // @ts-ignore - { googAudioMirroring: { exact: true } }, ]; - volume(volume: number) { if (!(0.0 <= volume && volume <= 1.0)) { throw Error('volume can only be in range [0.0, 1.0]'); @@ -38,7 +31,7 @@ export class HMSAudioTrackSettingsBuilder { if (maxBitrate && maxBitrate <= 0) { throw Error('maxBitrate should be >= 1'); } - this._maxBitrate = maxBitrate; + this._maxBitrate = this._audioMode === HMSAudioMode.MUSIC ? 320 : maxBitrate; return this; } @@ -48,13 +41,30 @@ export class HMSAudioTrackSettingsBuilder { return this; } + audioMode(mode: HMSAudioMode = HMSAudioMode.VOICE) { + this._audioMode = mode; + if (this._audioMode === HMSAudioMode.MUSIC) { + this._maxBitrate = 320; + } else { + this._maxBitrate = 32; + } + return this; + } + advanced(advanced: Array) { this._advanced = advanced; return this; } build() { - return new HMSAudioTrackSettings(this._volume, this._codec, this._maxBitrate, this._deviceId, this._advanced); + return new HMSAudioTrackSettings( + this._volume, + this._codec, + this._maxBitrate, + this._deviceId, + this._advanced, + this._audioMode, + ); } } @@ -64,6 +74,7 @@ export class HMSAudioTrackSettings implements IHMSAudioTrackSettings, IAnalytics readonly maxBitrate?: number; readonly deviceId?: string; readonly advanced?: Array; + readonly audioMode?: HMSAudioMode; constructor( volume?: number, @@ -71,18 +82,25 @@ export class HMSAudioTrackSettings implements IHMSAudioTrackSettings, IAnalytics maxBitrate?: number, deviceId?: string, advanced?: Array, + audioMode?: HMSAudioMode, ) { this.volume = volume; this.codec = codec; this.maxBitrate = maxBitrate; this.deviceId = deviceId; this.advanced = advanced; + this.audioMode = audioMode; + if (this.audioMode === HMSAudioMode.MUSIC) { + this.maxBitrate = 320; + } else { + this.maxBitrate = 32; + } } toConstraints(): MediaTrackConstraints { return { deviceId: this.deviceId, - advanced: this.advanced, + advanced: this.audioMode === HMSAudioMode.MUSIC ? [] : this.advanced, }; } diff --git a/packages/hms-video-store/src/media/settings/constants.ts b/packages/hms-video-store/src/media/settings/constants.ts new file mode 100644 index 0000000000..e1f0e8f4d2 --- /dev/null +++ b/packages/hms-video-store/src/media/settings/constants.ts @@ -0,0 +1,8 @@ +export const standardMediaConstraints = [ + { echoCancellation: { exact: true } }, + { highpassFilter: { exact: true } }, + { audioMirroring: { exact: true } }, + // These options can vary depending on the audio plugin + // { autoGainControl: { exact: true } }, + // { noiseSuppression: { exact: true } }, +]; diff --git a/packages/hms-video-store/src/media/streams/HMSLocalStream.ts b/packages/hms-video-store/src/media/streams/HMSLocalStream.ts index 10d9582df3..6b0b551e87 100644 --- a/packages/hms-video-store/src/media/streams/HMSLocalStream.ts +++ b/packages/hms-video-store/src/media/streams/HMSLocalStream.ts @@ -3,6 +3,7 @@ import HMSPublishConnection from '../../connection/publish/publishConnection'; import { SimulcastLayer } from '../../interfaces'; import HMSLogger from '../../utils/logger'; import { isNode } from '../../utils/support'; +import { HMSAudioTrackSettings, HMSVideoTrackSettings } from '../settings'; import { HMSLocalTrack, HMSLocalVideoTrack } from '../tracks'; export class HMSLocalStream extends HMSMediaStream { @@ -25,8 +26,11 @@ export class HMSLocalStream extends HMSMediaStream { return transceiver; } - async setMaxBitrateAndFramerate(track: HMSLocalTrack): Promise { - await this.connection?.setMaxBitrateAndFramerate(track); + async setMaxBitrateAndFramerate( + track: HMSLocalTrack, + updatedSettings?: HMSAudioTrackSettings | HMSVideoTrackSettings, + ): Promise { + await this.connection?.setMaxBitrateAndFramerate(track, updatedSettings); } // @ts-ignore diff --git a/packages/hms-video-store/src/media/tracks/HMSAudioTrack.ts b/packages/hms-video-store/src/media/tracks/HMSAudioTrack.ts index c3c75a824a..58fe7bc171 100644 --- a/packages/hms-video-store/src/media/tracks/HMSAudioTrack.ts +++ b/packages/hms-video-store/src/media/tracks/HMSAudioTrack.ts @@ -17,7 +17,8 @@ export class HMSAudioTrack extends HMSTrack { } getVolume() { - return this.audioElement ? this.audioElement.volume * 100 : null; + // floor is required because of floating-point precision. e.g 0.55*100 gives 55.00000000000001 + return this.audioElement ? Math.floor(this.audioElement.volume * 100) : null; } async setVolume(value: number) { diff --git a/packages/hms-video-store/src/media/tracks/HMSLocalAudioTrack.ts b/packages/hms-video-store/src/media/tracks/HMSLocalAudioTrack.ts index caef6b0629..69fd62e186 100644 --- a/packages/hms-video-store/src/media/tracks/HMSLocalAudioTrack.ts +++ b/packages/hms-video-store/src/media/tracks/HMSLocalAudioTrack.ts @@ -7,15 +7,15 @@ import { EventBus } from '../../events/EventBus'; import { HMSAudioTrackSettings as IHMSAudioTrackSettings } from '../../interfaces'; import { HMSAudioPlugin, HMSPluginSupportResult } from '../../plugins'; import { HMSAudioPluginsManager } from '../../plugins/audio'; +import Room from '../../sdk/models/HMSRoom'; import HMSLogger from '../../utils/logger'; -import { isBrowser, isIOS } from '../../utils/support'; import { getAudioTrack, isEmptyTrack } from '../../utils/track'; import { TrackAudioLevelMonitor } from '../../utils/track-audio-level-monitor'; import { HMSAudioTrackSettings, HMSAudioTrackSettingsBuilder } from '../settings'; import { HMSLocalStream } from '../streams'; function generateHasPropertyChanged(newSettings: Partial, oldSettings: HMSAudioTrackSettings) { - return function hasChanged(prop: 'codec' | 'volume' | 'maxBitrate' | 'deviceId' | 'advanced') { + return function hasChanged(prop: 'codec' | 'volume' | 'maxBitrate' | 'deviceId' | 'advanced' | 'audioMode') { return !isEqual(newSettings[prop], oldSettings[prop]); }; } @@ -26,6 +26,12 @@ export class HMSLocalAudioTrack extends HMSAudioTrack { private pluginsManager: HMSAudioPluginsManager; private processedTrack?: MediaStreamTrack; private manuallySelectedDeviceId?: string; + /** + * This is to keep track of all the tracks created so far and stop and clear them when creating new tracks to release microphone + * This is needed because when replaceTrackWith is called before updating native track, there is no way that track is available + * for you to stop, which leads to the microphone not released even after leave is called. + */ + private tracksCreated = new Set(); audioLevelMonitor?: TrackAudioLevelMonitor; @@ -46,9 +52,11 @@ export class HMSLocalAudioTrack extends HMSAudioTrack { source: string, private eventBus: EventBus, settings: HMSAudioTrackSettings = new HMSAudioTrackSettingsBuilder().build(), + private room?: Room, ) { super(stream, track, source); stream.tracks.push(this); + this.addTrackEventListeners(track); this.settings = settings; // Replace the 'default' or invalid deviceId with the actual deviceId @@ -56,13 +64,34 @@ export class HMSLocalAudioTrack extends HMSAudioTrack { if (settings.deviceId !== track.getSettings().deviceId && !isEmptyTrack(track)) { this.settings = this.buildNewSettings({ deviceId: track.getSettings().deviceId }); } - this.pluginsManager = new HMSAudioPluginsManager(this, eventBus); + this.pluginsManager = new HMSAudioPluginsManager(this, eventBus, room); this.setFirstTrackId(track.id); - if (isIOS() && isBrowser) { + if (source === 'regular') { document.addEventListener('visibilitychange', this.handleVisibilityChange); } } + clone(stream: HMSLocalStream) { + const track = new HMSLocalAudioTrack( + stream, + this.nativeTrack.clone(), + this.source!, + this.eventBus, + this.settings, + this.room, + ); + track.peerId = this.peerId; + + if (this.pluginsManager.pluginsMap.size > 0) { + this.pluginsManager.pluginsMap.forEach(value => { + track + .addPlugin(value) + .catch((e: Error) => HMSLogger.e(this.TAG, 'Plugin add failed while migrating', value, e)); + }); + } + return track; + } + getManuallySelectedDeviceId() { return this.manuallySelectedDeviceId; } @@ -72,11 +101,45 @@ export class HMSLocalAudioTrack extends HMSAudioTrack { } private handleVisibilityChange = async () => { - if (document.visibilityState === 'visible') { + // track state is fine do nothing + if (!this.shouldReacquireTrack()) { + HMSLogger.d(this.TAG, `visibiltiy: ${document.visibilityState}`, `${this}`); + return; + } + if (document.visibilityState === 'hidden') { + this.eventBus.analytics.publish( + this.sendInterruptionEvent({ + started: true, + reason: 'visibility-change', + }), + ); + } else { + HMSLogger.d(this.TAG, 'On visibile replacing track as it is not publishing'); await this.replaceTrackWith(this.settings); + this.eventBus.analytics.publish( + this.sendInterruptionEvent({ + started: false, + reason: 'visibility-change', + }), + ); } }; + /** + * Replace the new track in stream and update native track + * @param track + */ + private async updateTrack(track: MediaStreamTrack) { + track.enabled = this.enabled; + const localStream = this.stream as HMSLocalStream; + await localStream.replaceStreamTrack(this.nativeTrack, track); + // change nativeTrack so plugin can start its work + this.nativeTrack = track; + await this.replaceSenderTrack(); + const isLevelMonitored = Boolean(this.audioLevelMonitor); + isLevelMonitored && this.initAudioLevelMonitor(); + } + private async replaceTrackWith(settings: HMSAudioTrackSettings) { const prevTrack = this.nativeTrack; /* @@ -85,19 +148,21 @@ export class HMSLocalAudioTrack extends HMSAudioTrack { * no audio when the above getAudioTrack throws an error. ex: DeviceInUse error */ prevTrack?.stop(); - const isLevelMonitored = Boolean(this.audioLevelMonitor); + this.removeTrackEventListeners(prevTrack); + this.tracksCreated.forEach(track => track.stop()); + this.tracksCreated.clear(); try { const newTrack = await getAudioTrack(settings); - newTrack.enabled = this.enabled; + this.addTrackEventListeners(newTrack); + this.tracksCreated.add(newTrack); HMSLogger.d(this.TAG, 'replaceTrack, Previous track stopped', prevTrack, 'newTrack', newTrack); - - const localStream = this.stream as HMSLocalStream; - await localStream.replaceStreamTrack(prevTrack, newTrack); - // change nativeTrack so plugin can start its work - this.nativeTrack = newTrack; - await this.replaceSenderTrack(); - isLevelMonitored && this.initAudioLevelMonitor(); + await this.updateTrack(newTrack); } catch (e) { + // Generate a new track from previous settings so there will be audio because previous track is stopped + const newTrack = await getAudioTrack(this.settings); + this.addTrackEventListeners(newTrack); + this.tracksCreated.add(newTrack); + await this.updateTrack(newTrack); if (this.isPublished) { this.eventBus.analytics.publish( AnalyticsEventFactory.publish({ @@ -119,8 +184,8 @@ export class HMSLocalAudioTrack extends HMSAudioTrack { return; } - // Replace silent empty track with an actual audio track, if enabled. - if (value && isEmptyTrack(this.nativeTrack)) { + // Replace silent empty track or muted track(happens when microphone is disabled from address bar in iOS) with an actual audio track, if enabled or ended track or when silence is detected. + if (value && this.shouldReacquireTrack()) { await this.replaceTrackWith(this.settings); } await super.setEnabled(value); @@ -216,11 +281,11 @@ export class HMSLocalAudioTrack extends HMSAudioTrack { await this.pluginsManager.closeContext(); this.transceiver = undefined; this.processedTrack?.stop(); + this.tracksCreated.forEach(track => track.stop()); + this.tracksCreated.clear(); this.isPublished = false; this.destroyAudioLevelMonitor(); - if (isIOS() && isBrowser) { - document.removeEventListener('visibilitychange', this.handleVisibilityChange); - } + document.removeEventListener('visibilitychange', this.handleVisibilityChange); } /** @@ -238,6 +303,42 @@ export class HMSLocalAudioTrack extends HMSAudioTrack { return this.processedTrack || this.nativeTrack; } + private addTrackEventListeners(track: MediaStreamTrack) { + track.addEventListener('mute', this.handleTrackMute); + track.addEventListener('unmute', this.handleTrackUnmute); + } + + private removeTrackEventListeners(track: MediaStreamTrack) { + track.removeEventListener('mute', this.handleTrackMute); + track.removeEventListener('unmute', this.handleTrackUnmute); + } + + private handleTrackMute = () => { + HMSLogger.d(this.TAG, 'muted natively'); + const reason = document.visibilityState === 'hidden' ? 'visibility-change' : 'incoming-call'; + this.eventBus.analytics.publish( + this.sendInterruptionEvent({ + started: true, + reason, + }), + ); + }; + + /** @internal */ + handleTrackUnmute = async () => { + HMSLogger.d(this.TAG, 'unmuted natively'); + const reason = document.visibilityState === 'hidden' ? 'visibility-change' : 'incoming-call'; + this.eventBus.analytics.publish( + this.sendInterruptionEvent({ + started: false, + reason, + }), + ); + await this.setEnabled(this.enabled); + // whatsapp call doesn't seem to send video unmute natively, so use audio unmute to play video + this.eventBus.localAudioUnmutedNatively.publish(); + }; + private replaceSenderTrack = async () => { if (!this.transceiver || this.transceiver.direction !== 'sendonly') { HMSLogger.d(this.TAG, `transceiver for ${this.trackId} not available or not connected yet`); @@ -246,20 +347,26 @@ export class HMSLocalAudioTrack extends HMSAudioTrack { await this.transceiver.sender.replaceTrack(this.processedTrack || this.nativeTrack); }; + private shouldReacquireTrack = () => { + return ( + isEmptyTrack(this.nativeTrack) || this.isTrackNotPublishing() || this.audioLevelMonitor?.isSilentThisInstant() + ); + }; + private buildNewSettings(settings: Partial) { - const { volume, codec, maxBitrate, deviceId, advanced } = { ...this.settings, ...settings }; - const newSettings = new HMSAudioTrackSettings(volume, codec, maxBitrate, deviceId, advanced); + const { volume, codec, maxBitrate, deviceId, advanced, audioMode } = { ...this.settings, ...settings }; + const newSettings = new HMSAudioTrackSettings(volume, codec, maxBitrate, deviceId, advanced, audioMode); return newSettings; } private handleSettingsChange = async (settings: HMSAudioTrackSettings) => { const stream = this.stream as HMSLocalStream; const hasPropertyChanged = generateHasPropertyChanged(settings, this.settings); - if (hasPropertyChanged('maxBitrate') && settings.maxBitrate) { - await stream.setMaxBitrateAndFramerate(this); + if ((hasPropertyChanged('maxBitrate') || hasPropertyChanged('audioMode')) && settings.maxBitrate) { + await stream.setMaxBitrateAndFramerate(this, settings); } - if (hasPropertyChanged('advanced')) { + if (hasPropertyChanged('advanced') || hasPropertyChanged('audioMode')) { await this.replaceTrackWith(settings); } }; @@ -273,6 +380,14 @@ export class HMSLocalAudioTrack extends HMSAudioTrack { const hasPropertyChanged = generateHasPropertyChanged(settings, this.settings); if (hasPropertyChanged('deviceId')) { this.manuallySelectedDeviceId = !internal ? settings.deviceId : this.manuallySelectedDeviceId; + HMSLogger.d( + this.TAG, + 'device change', + 'manual selection:', + this.manuallySelectedDeviceId, + 'new device:', + settings.deviceId, + ); await this.replaceTrackWith(settings); const groupId = this.nativeTrack.getSettings().groupId; if (!internal && settings.deviceId) { diff --git a/packages/hms-video-store/src/media/tracks/HMSLocalVideoTrack.ts b/packages/hms-video-store/src/media/tracks/HMSLocalVideoTrack.ts index 31218fc3c5..9995f303ed 100644 --- a/packages/hms-video-store/src/media/tracks/HMSLocalVideoTrack.ts +++ b/packages/hms-video-store/src/media/tracks/HMSLocalVideoTrack.ts @@ -16,7 +16,9 @@ import { HMSPluginSupportResult, HMSVideoPlugin } from '../../plugins'; import { HMSMediaStreamPlugin, HMSVideoPluginsManager } from '../../plugins/video'; import { HMSMediaStreamPluginsManager } from '../../plugins/video/HMSMediaStreamPluginsManager'; import { LocalTrackManager } from '../../sdk/LocalTrackManager'; +import Room from '../../sdk/models/HMSRoom'; import HMSLogger from '../../utils/logger'; +import { isBrowser, isMobile } from '../../utils/support'; import { getVideoTrack, isEmptyTrack } from '../../utils/track'; import { HMSVideoTrackSettings, HMSVideoTrackSettingsBuilder } from '../settings'; import { HMSLocalStream } from '../streams'; @@ -36,6 +38,7 @@ export class HMSLocalVideoTrack extends HMSVideoTrack { private processedTrack?: MediaStreamTrack; private _layerDefinitions: HMSSimulcastLayerDefinition[] = []; private TAG = '[HMSLocalVideoTrack]'; + private enabledStateBeforeBackground = false; /** * true if it's screenshare and current tab is what is being shared. Browser dependent, Chromium only @@ -67,8 +70,10 @@ export class HMSLocalVideoTrack extends HMSVideoTrack { source: string, private eventBus: EventBus, settings: HMSVideoTrackSettings = new HMSVideoTrackSettingsBuilder().build(), + private room?: Room, ) { super(stream, track, source); + this.addTrackEventListeners(track); stream.tracks.push(this); this.setVideoHandler(new VideoElementManager(this)); this.settings = settings; @@ -78,8 +83,36 @@ export class HMSLocalVideoTrack extends HMSVideoTrack { this.settings = this.buildNewSettings({ deviceId: track.getSettings().deviceId }); } this.pluginsManager = new HMSVideoPluginsManager(this, eventBus); - this.mediaStreamPluginsManager = new HMSMediaStreamPluginsManager(eventBus); + this.mediaStreamPluginsManager = new HMSMediaStreamPluginsManager(eventBus, room); this.setFirstTrackId(this.trackId); + this.eventBus.localAudioUnmutedNatively.subscribe(this.handleTrackUnmute); + if (isBrowser && source === 'regular' && isMobile()) { + document.addEventListener('visibilitychange', this.handleVisibilityChange); + } + } + + clone(stream: HMSLocalStream) { + const track = new HMSLocalVideoTrack( + stream, + this.nativeTrack.clone(), + this.source!, + this.eventBus, + this.settings, + this.room, + ); + track.peerId = this.peerId; + + if (this.pluginsManager.pluginsMap.size > 0) { + this.pluginsManager.pluginsMap.forEach(value => { + track + .addPlugin(value) + .catch((e: Error) => HMSLogger.e(this.TAG, 'Plugin add failed while migrating', value, e)); + }); + } + if (this.mediaStreamPluginsManager.plugins.size > 0) { + track.addStreamPlugins(Array.from(this.mediaStreamPluginsManager.plugins)); + } + return track; } /** @internal */ @@ -99,6 +132,7 @@ export class HMSLocalVideoTrack extends HMSVideoTrack { * use this function to set the enabled state of a track. If true the track will be unmuted and muted otherwise. * @param value */ + // eslint-disable-next-line complexity async setEnabled(value: boolean): Promise { if (value === this.enabled) { return; @@ -137,6 +171,7 @@ export class HMSLocalVideoTrack extends HMSVideoTrack { } else { await this.setProcessedTrack(); } + this.videoHandler.updateSinks(); } catch (e) { console.error('error in processing plugin(s)', e); } @@ -227,11 +262,18 @@ export class HMSLocalVideoTrack extends HMSVideoTrack { * @internal */ async cleanup() { + this.eventBus.localAudioUnmutedNatively.unsubscribe(this.handleTrackUnmute); + this.removeTrackEventListeners(this.nativeTrack); + // Stopping the plugin before cleaning the track is more predictable when dealing with 3rd party plugins + await this.mediaStreamPluginsManager.cleanup(); + await this.pluginsManager.cleanup(); super.cleanup(); this.transceiver = undefined; - await this.pluginsManager.cleanup(); this.processedTrack?.stop(); this.isPublished = false; + if (isBrowser && isMobile()) { + document.removeEventListener('visibilitychange', this.handleVisibilityChange); + } } /** @@ -338,9 +380,11 @@ export class HMSLocalVideoTrack extends HMSVideoTrack { * you are requesting for a new device. * Note: Do not change the order of this. */ + this.removeTrackEventListeners(prevTrack); prevTrack?.stop(); try { const newTrack = await getVideoTrack(settings); + this.addTrackEventListeners(newTrack); HMSLogger.d(this.TAG, 'replaceTrack, Previous track stopped', prevTrack, 'newTrack', newTrack); // Replace deviceId with actual deviceId when it is default if (this.settings.deviceId === 'default') { @@ -348,6 +392,13 @@ export class HMSLocalVideoTrack extends HMSVideoTrack { } return newTrack; } catch (error) { + // Generate a new track from previous settings so there won't be blank tile because previous track is stopped + const track = await getVideoTrack(this.settings); + this.addTrackEventListeners(track); + await this.replaceSender(track, this.enabled); + this.nativeTrack = track; + await this.processPlugins(); + this.videoHandler.updateSinks(); if (this.isPublished) { this.eventBus.analytics.publish( AnalyticsEventFactory.publish({ @@ -367,6 +418,8 @@ export class HMSLocalVideoTrack extends HMSVideoTrack { private async replaceTrackWithBlank() { const prevTrack = this.nativeTrack; const newTrack = LocalTrackManager.getEmptyVideoTrack(prevTrack); + this.removeTrackEventListeners(prevTrack); + this.addTrackEventListeners(newTrack); prevTrack?.stop(); HMSLogger.d(this.TAG, 'replaceTrackWithBlank, Previous track stopped', prevTrack, 'newTrack', newTrack); return newTrack; @@ -463,6 +516,45 @@ export class HMSLocalVideoTrack extends HMSVideoTrack { } }; + private addTrackEventListeners(track: MediaStreamTrack) { + track.addEventListener('mute', this.handleTrackMute); + track.addEventListener('unmute', this.handleTrackUnmuteNatively); + } + + private removeTrackEventListeners(track: MediaStreamTrack) { + track.removeEventListener('mute', this.handleTrackMute); + track.removeEventListener('unmute', this.handleTrackUnmuteNatively); + } + + private handleTrackMute = () => { + HMSLogger.d(this.TAG, 'muted natively', document.visibilityState); + const reason = document.visibilityState === 'hidden' ? 'visibility-change' : 'incoming-call'; + this.eventBus.analytics.publish( + this.sendInterruptionEvent({ + started: true, + reason: reason, + }), + ); + this.eventBus.localVideoEnabled.publish({ enabled: false, track: this }); + }; + + /** @internal */ + handleTrackUnmuteNatively = async () => { + HMSLogger.d(this.TAG, 'unmuted natively'); + const reason = document.visibilityState === 'hidden' ? 'visibility-change' : 'incoming-call'; + + this.eventBus.analytics.publish( + this.sendInterruptionEvent({ + started: false, + reason: reason, + }), + ); + this.handleTrackUnmute(); + this.eventBus.localVideoEnabled.publish({ enabled: this.enabled, track: this }); + this.eventBus.localVideoUnmutedNatively.publish(); + await this.setEnabled(this.enabled); + }; + /** * This will either remove or update the processedTrack value on the class instance. * It will also replace sender if the processedTrack is updated @@ -477,4 +569,33 @@ export class HMSLocalVideoTrack extends HMSVideoTrack { } await this.replaceSenderTrack(this.processedTrack || this.nativeTrack); }; + + // eslint-disable-next-line complexity + private handleVisibilityChange = async () => { + if (document.visibilityState === 'hidden') { + this.enabledStateBeforeBackground = this.enabled; + if (this.enabled) { + await this.setEnabled(false); + } + // started interruption event + this.eventBus.analytics.publish( + this.sendInterruptionEvent({ + started: true, + reason: 'visibility-change', + }), + ); + } else { + HMSLogger.d(this.TAG, 'visibility visible, restoring track state', this.enabledStateBeforeBackground); + if (this.enabledStateBeforeBackground) { + await this.setEnabled(true); + } + // ended interruption event + this.eventBus.analytics.publish( + this.sendInterruptionEvent({ + started: false, + reason: 'visibility-change', + }), + ); + } + }; } diff --git a/packages/hms-video-store/src/media/tracks/HMSRemoteVideoTrack.ts b/packages/hms-video-store/src/media/tracks/HMSRemoteVideoTrack.ts index 15edafbf5a..abff98bb41 100644 --- a/packages/hms-video-store/src/media/tracks/HMSRemoteVideoTrack.ts +++ b/packages/hms-video-store/src/media/tracks/HMSRemoteVideoTrack.ts @@ -18,9 +18,11 @@ export class HMSRemoteVideoTrack extends HMSVideoTrack { private history = new TrackHistory(); private preferredLayer: HMSPreferredSimulcastLayer = HMSSimulcastLayer.HIGH; private bizTrackId!: string; + private disableNoneLayerRequest = false; - constructor(stream: HMSRemoteStream, track: MediaStreamTrack, source?: string) { + constructor(stream: HMSRemoteStream, track: MediaStreamTrack, source?: string, disableNoneLayerRequest?: boolean) { super(stream, track, source); + this.disableNoneLayerRequest = !!disableNoneLayerRequest; this.setVideoHandler(new VideoElementManager(this)); } @@ -89,6 +91,10 @@ export class HMSRemoteVideoTrack extends HMSVideoTrack { return this.preferredLayer; } + getPreferredLayerDefinition() { + return this._layerDefinitions.find(layer => layer.layer === this.preferredLayer); + } + replaceTrack(track: HMSRemoteVideoTrack) { this.nativeTrack = track.nativeTrack; if (track.transceiver) { @@ -142,10 +148,7 @@ export class HMSRemoteVideoTrack extends HMSVideoTrack { * @returns {boolean} isDegraded - returns true if degraded * */ setLayerFromServer(layerUpdate: VideoTrackLayerUpdate) { - this._degraded = - this.enabled && - (layerUpdate.publisher_degraded || layerUpdate.subscriber_degraded) && - layerUpdate.current_layer === HMSSimulcastLayer.NONE; + this._degraded = this.getDegradationValue(layerUpdate); this._degradedAt = this._degraded ? new Date() : this._degradedAt; const currentLayer = layerUpdate.current_layer; HMSLogger.d( @@ -165,8 +168,22 @@ export class HMSRemoteVideoTrack extends HMSVideoTrack { return this._degraded; } + private getDegradationValue(layerUpdate: VideoTrackLayerUpdate) { + return ( + this.enabled && + (layerUpdate.publisher_degraded || layerUpdate.subscriber_degraded) && + layerUpdate.current_layer === HMSSimulcastLayer.NONE + ); + } + private async updateLayer(source: string) { - const newLayer = this.degraded || !this.enabled || !this.hasSinks() ? HMSSimulcastLayer.NONE : this.preferredLayer; + let newLayer: HMSSimulcastLayer = this.preferredLayer; + if (this.enabled && this.hasSinks()) { + newLayer = this.preferredLayer; + // send none only when the flag is not set + } else if (!this.disableNoneLayerRequest) { + newLayer = HMSSimulcastLayer.NONE; + } if (!this.shouldSendVideoLayer(newLayer, source)) { return; } diff --git a/packages/hms-video-store/src/media/tracks/HMSTrack.ts b/packages/hms-video-store/src/media/tracks/HMSTrack.ts index b3c5574bb1..208d021ba4 100644 --- a/packages/hms-video-store/src/media/tracks/HMSTrack.ts +++ b/packages/hms-video-store/src/media/tracks/HMSTrack.ts @@ -1,4 +1,5 @@ import { HMSTrackType } from './HMSTrackType'; +import AnalyticsEventFactory from '../../analytics/AnalyticsEventFactory'; import { stringifyMediaStreamTrack } from '../../utils/json'; import HMSLogger from '../../utils/logger'; import { HMSMediaStream } from '../streams'; @@ -85,6 +86,25 @@ export abstract class HMSTrack { this.firstTrackId = trackId; } + isTrackNotPublishing = () => { + return this.nativeTrack.readyState === 'ended' || this.nativeTrack.muted; + }; + + /** + * @internal + * It will send event to analytics when interruption start/stop + */ + sendInterruptionEvent({ started, reason }: { started: boolean; reason: string }) { + return AnalyticsEventFactory.interruption({ + started, + type: this.type, + reason, + deviceInfo: { + deviceId: this.nativeTrack.getSettings().deviceId, + groupId: this.nativeTrack.getSettings().groupId, + }, + }); + } /** * @internal * take care of - diff --git a/packages/hms-video-store/src/media/tracks/HMSTrackExceptionTrackType.ts b/packages/hms-video-store/src/media/tracks/HMSTrackExceptionTrackType.ts new file mode 100644 index 0000000000..5bca9518b2 --- /dev/null +++ b/packages/hms-video-store/src/media/tracks/HMSTrackExceptionTrackType.ts @@ -0,0 +1,6 @@ +export enum HMSTrackExceptionTrackType { + AUDIO = 'audio', + VIDEO = 'video', + AUDIO_VIDEO = 'audio, video', + SCREEN = 'screen', +} diff --git a/packages/hms-video-store/src/media/tracks/HMSVideoTrack.ts b/packages/hms-video-store/src/media/tracks/HMSVideoTrack.ts index ced5ef3e90..8e577d4579 100644 --- a/packages/hms-video-store/src/media/tracks/HMSVideoTrack.ts +++ b/packages/hms-video-store/src/media/tracks/HMSVideoTrack.ts @@ -1,6 +1,8 @@ import { HMSTrack, HMSTrackSource } from './HMSTrack'; import { HMSTrackType } from './HMSTrackType'; import { VideoElementManager } from './VideoElementManager'; +import HMSLogger from '../../utils/logger'; +import { isSafari } from '../../utils/support'; import { HMSMediaStream } from '../streams'; export class HMSVideoTrack extends HMSTrack { @@ -77,13 +79,38 @@ export class HMSVideoTrack extends HMSTrack { this.reduceSinkCount(); } } - videoElement.srcObject = new MediaStream([track]); + + this.addPropertiesToElement(videoElement); + const stream = new MediaStream([track]); + videoElement.srcObject = stream; + this.reTriggerPlay({ videoElement }); this.sinkCount++; } + handleTrackUnmute = () => { + this.getSinks().forEach(videoElement => this.reTriggerPlay({ videoElement })); + }; + + private reTriggerPlay = ({ videoElement }: { videoElement: HTMLVideoElement }) => { + setTimeout(() => { + videoElement.play().catch((e: Error) => { + HMSLogger.w('[HMSVideoTrack]', 'failed to play', e.message); + }); + }, 0); + }; + private reduceSinkCount() { if (this.sinkCount > 0) { this.sinkCount--; } } + + private addPropertiesToElement(element: HTMLVideoElement) { + if (!isSafari) { + element.autoplay = true; + } + element.playsInline = true; + element.muted = true; + element.controls = false; + } } diff --git a/packages/hms-video-store/src/media/tracks/RemoteVideoTrack.test.ts b/packages/hms-video-store/src/media/tracks/RemoteVideoTrack.test.ts index 33729c1c3b..70fde7a17f 100644 --- a/packages/hms-video-store/src/media/tracks/RemoteVideoTrack.test.ts +++ b/packages/hms-video-store/src/media/tracks/RemoteVideoTrack.test.ts @@ -19,7 +19,7 @@ describe('remoteVideoTrack', () => { const connection = { sendOverApiDataChannelWithResponse } as unknown as HMSSubscribeConnection; stream = new HMSRemoteStream(nativeStream, connection); nativeTrack = { id: trackId, kind: 'video', enabled: true } as MediaStreamTrack; - track = new HMSRemoteVideoTrack(stream, nativeTrack, 'regular'); + track = new HMSRemoteVideoTrack(stream, nativeTrack, 'regular', false); window.MediaStream = jest.fn().mockImplementation(() => ({ addTrack: jest.fn(), // Add any method you want to mock @@ -156,3 +156,63 @@ describe('remoteVideoTrack', () => { expectLayersSent([HMSSimulcastLayer.HIGH, HMSSimulcastLayer.NONE]); }); }); + +describe('HMSRemoteVideoTrack with disableNoneLayerRequest', () => { + let stream: HMSRemoteStream; + let sendOverApiDataChannelWithResponse: jest.Mock; + let track: HMSRemoteVideoTrack; + let nativeTrack: MediaStreamTrack; + let videoElement: HTMLVideoElement; + const trackId = 'test-track-id'; + + beforeEach(() => { + videoElement = document.createElement('video'); + sendOverApiDataChannelWithResponse = jest.fn(); + const connection = { sendOverApiDataChannelWithResponse } as unknown as HMSSubscribeConnection; + const nativeStream = new MediaStream(); + stream = new HMSRemoteStream(nativeStream, connection); + nativeTrack = { id: trackId, kind: 'video', enabled: true } as MediaStreamTrack; + track = new HMSRemoteVideoTrack(stream, nativeTrack, 'regular', true); // disableNoneLayerRequest flag is set + track.setTrackId(trackId); + + window.MediaStream = jest.fn().mockImplementation(() => ({ + addTrack: jest.fn(), + })); + }); + + const expectLayersSent = (layers: HMSSimulcastLayer[]) => { + const allCalls = sendOverApiDataChannelWithResponse.mock.calls; + expect(allCalls.length).toBe(layers.length); + for (let i = 0; i < allCalls.length; i++) { + const data = allCalls[i][0]; + expect(data.params.max_spatial_layer).toBe(layers[i]); + } + }; + + const sfuDegrades = () => { + track.setLayerFromServer({ + subscriber_degraded: true, + expected_layer: HMSSimulcastLayer.HIGH, + current_layer: HMSSimulcastLayer.NONE, + publisher_degraded: false, + track_id: trackId, + }); + }; + + test('disableNoneLayerRequest - degradation', async () => { + await track.addSink(videoElement); + expectLayersSent([HMSSimulcastLayer.HIGH]); + + sfuDegrades(); + expectLayersSent([HMSSimulcastLayer.HIGH]); + }); + + test('disableNoneLayerRequest - mute and removeSink', async () => { + await track.addSink(videoElement); + track.setEnabled(false); + expectLayersSent([HMSSimulcastLayer.HIGH]); + + await track.removeSink(videoElement); + expectLayersSent([HMSSimulcastLayer.HIGH]); + }); +}); diff --git a/packages/hms-video-store/src/media/tracks/videoElementManager.test.ts b/packages/hms-video-store/src/media/tracks/videoElementManager.test.ts index 6e00b464fa..c8dcad2ba5 100644 --- a/packages/hms-video-store/src/media/tracks/videoElementManager.test.ts +++ b/packages/hms-video-store/src/media/tracks/videoElementManager.test.ts @@ -36,6 +36,7 @@ describe('videoElementManager', () => { kind: 'video', enabled: true, getSettings: jest.fn(() => ({})), + addEventListener: jest.fn(() => {}), } as unknown as MediaStreamTrack; localTrack = new HMSLocalVideoTrack(localStream, localNativeTrack, 'regular', new EventBus()); @@ -43,7 +44,12 @@ describe('videoElementManager', () => { const connection = { sendOverApiDataChannelWithResponse } as unknown as HMSSubscribeConnection; remoteNativeStream = { id: remoteStreamId } as MediaStream; remoteStream = new HMSRemoteStream(remoteNativeStream, connection); - remoteNativeTrack = { id: remoteTrackId, kind: 'video', enabled: true } as MediaStreamTrack; + remoteNativeTrack = { + id: remoteTrackId, + kind: 'video', + enabled: true, + addEventListener: jest.fn(() => {}), + } as unknown as MediaStreamTrack; remoteTrack = new HMSRemoteVideoTrack(remoteStream, remoteNativeTrack, 'regular'); window.MediaStream = jest.fn().mockImplementation(() => ({ addTrack: jest.fn(), diff --git a/packages/hms-video-store/src/notification-manager/HMSNotificationMethod.ts b/packages/hms-video-store/src/notification-manager/HMSNotificationMethod.ts index 72c5d06921..c5a287dde2 100644 --- a/packages/hms-video-store/src/notification-manager/HMSNotificationMethod.ts +++ b/packages/hms-video-store/src/notification-manager/HMSNotificationMethod.ts @@ -28,11 +28,13 @@ export enum HMSNotificationMethod { RTMP_UPDATE = 'on-rtmp-update', RECORDING_UPDATE = 'on-record-update', HLS_UPDATE = 'on-hls-update', + TRANSCRIPTION_UPDATE = 'on-transcription-update', METADATA_CHANGE = 'on-metadata-change', POLL_START = 'on-poll-start', POLL_STOP = 'on-poll-stop', POLL_STATS = 'on-poll-stats', ROOM_INFO = 'room-info', SESSION_INFO = 'session-info', + NODE_INFO = 'node-info', WHITEBOARD_UPDATE = 'on-whiteboard-update', } diff --git a/packages/hms-video-store/src/notification-manager/HMSNotifications.ts b/packages/hms-video-store/src/notification-manager/HMSNotifications.ts index 1e5959ef70..11fffbb254 100644 --- a/packages/hms-video-store/src/notification-manager/HMSNotifications.ts +++ b/packages/hms-video-store/src/notification-manager/HMSNotifications.ts @@ -1,4 +1,5 @@ import { VideoTrackLayerUpdate } from '../connection/channel-messages'; +import { HMSPeerType } from '../interfaces/peer/hms-peer'; import { HMSRole } from '../interfaces/role'; import { HMSLocalTrack } from '../media/tracks'; import { HMSTrack, HMSTrackSource } from '../media/tracks/HMSTrack'; @@ -39,6 +40,14 @@ export interface Info { name: string; data: string; user_id: string; + type: HMSPeerType; +} + +export interface FindPeerByNameInfo { + name: string; + peer_id: string; + role: string; + type: HMSPeerType; } export enum HMSRecordingState { @@ -59,7 +68,16 @@ export enum HMSStreamingState { FAILED = 'failed', } -interface PluginPermissions { +export enum HMSTranscriptionState { + INITIALISED = 'initialised', + STARTED = 'started', + STOPPED = 'stopped', + FAILED = 'failed', +} +export enum HMSTranscriptionMode { + CAPTION = 'caption', +} +export interface WhiteBoardPluginPermissions { permissions?: { // list of roles admin?: Array; @@ -68,13 +86,32 @@ interface PluginPermissions { }; } +export interface TranscriptionPluginPermissions { + permissions?: { + // list of roles + admin?: Array; + }; + mode: HMSTranscriptionMode; +} + +export interface NoiseCancellationPlugin { + enabled?: boolean; +} +export enum Plugins { + WHITEBOARD = 'whiteboard', + TRANSCRIPTIONS = 'transcriptions', + NOISE_CANCELLATION = 'noiseCancellation', +} + export interface PolicyParams { name: string; known_roles: { [role: string]: HMSRole; }; plugins: { - [plugin in 'whiteboard']?: PluginPermissions; + [Plugins.WHITEBOARD]?: WhiteBoardPluginPermissions; + [Plugins.TRANSCRIPTIONS]?: TranscriptionPluginPermissions[]; + [Plugins.NOISE_CANCELLATION]?: NoiseCancellationPlugin; }; template_id: string; app_data?: Record; @@ -120,6 +157,17 @@ export interface PeerNotification { is_from_room_state?: boolean; } +export interface TranscriptionNotification { + state?: HMSTranscriptionState; + mode?: HMSTranscriptionMode; + initialised_at?: number; + started_at?: number; + updated_at?: number; + stopped_at?: number; + peer?: PeerNotificationInfo; + error?: ServerError; +} + export interface RoomState { name: string; session_id?: string; @@ -151,6 +199,7 @@ export interface RoomState { rtmp: { enabled: boolean; started_at?: number; state?: HMSStreamingState }; hls: HLSNotification; }; + transcriptions?: TranscriptionNotification[]; } export interface PeerListNotification { @@ -279,13 +328,24 @@ export interface HLSNotification { hls_recording?: HLSRecording; } +export enum HLSPlaylistType { + DVR = 'dvr', + NO_DVR = 'no-dvr', +} +export enum HLSStreamType { + REGULAR = 'regular', + SCREEN = 'screen', + COMPOSITE = 'composite', +} export interface HLSVariantInfo { url: string; meeting_url?: string; + playlist_type?: HLSPlaylistType; metadata?: string; started_at?: number; initialised_at?: number; state?: HMSStreamingState; + stream_type?: HLSStreamType; } export interface MetadataChangeNotification { @@ -333,3 +393,7 @@ export interface WhiteboardInfo { state?: string; attributes?: Array<{ name: string; value: unknown }>; } + +export interface NodeInfo { + sfu_node_id: string; +} diff --git a/packages/hms-video-store/src/notification-manager/NotificationManager.ts b/packages/hms-video-store/src/notification-manager/NotificationManager.ts index cb6922b20a..64fcf9c314 100644 --- a/packages/hms-video-store/src/notification-manager/NotificationManager.ts +++ b/packages/hms-video-store/src/notification-manager/NotificationManager.ts @@ -14,6 +14,7 @@ import { WhiteboardManager } from './managers/WhiteboardManager'; import { HMSNotificationMethod } from './HMSNotificationMethod'; import { ConnectionQualityList, + NodeInfo, OnTrackLayerUpdateNotification, PolicyParams, SpeakerList, @@ -168,6 +169,10 @@ export class NotificationManager { this.policyChangeManager.handlePolicyChange(notification as PolicyParams); break; + case HMSNotificationMethod.NODE_INFO: + this.transport.setSFUNodeId((notification as NodeInfo).sfu_node_id); + break; + default: break; } diff --git a/packages/hms-video-store/src/notification-manager/fixtures.ts b/packages/hms-video-store/src/notification-manager/fixtures.ts index c02f5f972f..5bb8fe4485 100644 --- a/packages/hms-video-store/src/notification-manager/fixtures.ts +++ b/packages/hms-video-store/src/notification-manager/fixtures.ts @@ -1,10 +1,11 @@ import { MessageNotification, PeerListNotification, PeerNotification, SpeakerList } from './HMSNotifications'; +import { HMSPeerType } from '../interfaces'; export const FAKE_PEER_ID = 'peer_id_1'; export const fakePeer: PeerNotification = { peer_id: 'peer_id_0', - info: { data: 'data', name: 'Sarvesh0', user_id: 'customer_user_id' }, + info: { data: 'data', name: 'Sarvesh0', user_id: 'customer_user_id', type: HMSPeerType.REGULAR }, role: 'host', tracks: {}, groups: [], @@ -17,6 +18,7 @@ export const fakePeerList: PeerListNotification = { name: 'Sarvesh1', data: 'data', user_id: 'customer_user_id', + type: HMSPeerType.REGULAR, }, role: 'host', peer_id: FAKE_PEER_ID, @@ -32,7 +34,7 @@ export const fakePeerList: PeerListNotification = { track_id_2: { mute: false, type: 'video', - source: 'regular', + source: HMSPeerType.REGULAR, description: '', track_id: 'track_id_2', stream_id: 'stream_id_1', @@ -45,6 +47,7 @@ export const fakePeerList: PeerListNotification = { name: 'Sarvesh3', data: 'data', user_id: 'customer_user_id', + type: HMSPeerType.REGULAR, }, peer_id: 'peer_id_3', role: 'viewer', @@ -85,6 +88,7 @@ export const fakeReconnectPeerList: PeerListNotification = { name: 'Sarvesh1', data: 'data', user_id: 'customer_user_id', + type: HMSPeerType.REGULAR, }, role: 'host', peer_id: FAKE_PEER_ID, @@ -100,7 +104,7 @@ export const fakeReconnectPeerList: PeerListNotification = { track_id_2: { mute: false, type: 'video', - source: 'regular', + source: HMSPeerType.REGULAR, description: '', track_id: 'track_id_2', stream_id: 'stream_id_1', @@ -113,6 +117,7 @@ export const fakeReconnectPeerList: PeerListNotification = { name: 'Sarvesh2', data: 'data', user_id: 'customer_user_id', + type: HMSPeerType.REGULAR, }, peer_id: 'peer_id_2', role: 'viewer', diff --git a/packages/hms-video-store/src/notification-manager/managers/PeerListManager.ts b/packages/hms-video-store/src/notification-manager/managers/PeerListManager.ts index 5d0f46f6c9..021f9ca54e 100644 --- a/packages/hms-video-store/src/notification-manager/managers/PeerListManager.ts +++ b/packages/hms-video-store/src/notification-manager/managers/PeerListManager.ts @@ -98,6 +98,7 @@ export class PeerListManager { name: peer.name, data: peer.metadata || '', user_id: peer.customerUserId || '', + type: peer.type, }, tracks: {}, groups: [], diff --git a/packages/hms-video-store/src/notification-manager/managers/PollsManager.ts b/packages/hms-video-store/src/notification-manager/managers/PollsManager.ts index 94d6820ee1..3c32618e9e 100644 --- a/packages/hms-video-store/src/notification-manager/managers/PollsManager.ts +++ b/packages/hms-video-store/src/notification-manager/managers/PollsManager.ts @@ -88,8 +88,6 @@ export class PollsManager { } this.updatePollResult(savedPoll, updatedPoll); - await this.updatePollResponses(savedPoll, false); - updatedPolls.push(savedPoll); } diff --git a/packages/hms-video-store/src/notification-manager/managers/RoomUpdateManager.ts b/packages/hms-video-store/src/notification-manager/managers/RoomUpdateManager.ts index 90a66cf6e1..96d3b71ffc 100644 --- a/packages/hms-video-store/src/notification-manager/managers/RoomUpdateManager.ts +++ b/packages/hms-video-store/src/notification-manager/managers/RoomUpdateManager.ts @@ -7,6 +7,7 @@ import { HMSHLSRecording, HMSRoomUpdate, HMSSFURecording, + HMSTranscriptionInfo, HMSUpdateListener, } from '../../interfaces'; import { ServerError } from '../../interfaces/internal'; @@ -26,6 +27,7 @@ import { RoomState, RTMPNotification, SessionInfo, + TranscriptionNotification, } from '../HMSNotifications'; export class RoomUpdateManager { @@ -57,6 +59,9 @@ export class RoomUpdateManager { case HMSNotificationMethod.HLS_UPDATE: this.updateHLSStatus(notification as HLSNotification); break; + case HMSNotificationMethod.TRANSCRIPTION_UPDATE: + this.handleTranscriptionStatus([notification as TranscriptionNotification]); + break; default: break; } @@ -93,7 +98,7 @@ export class RoomUpdateManager { } private onRoomState(roomNotification: RoomState) { - const { recording, streaming, session_id, started_at, name } = roomNotification; + const { recording, streaming, transcriptions, session_id, started_at, name } = roomNotification; const room = this.store.getRoom(); if (!room) { HMSLogger.w(this.TAG, 'on room state - room not present'); @@ -112,11 +117,29 @@ export class RoomUpdateManager { room.hls = this.convertHls(streaming?.hls); + room.transcriptions = this.addTranscriptionDetail(transcriptions); + room.sessionId = session_id; room.startedAt = convertDateNumToDate(started_at); this.listener?.onRoomUpdate(HMSRoomUpdate.RECORDING_STATE_UPDATED, room); } + private addTranscriptionDetail(transcriptions?: TranscriptionNotification[]): HMSTranscriptionInfo[] { + if (!transcriptions) { + return []; + } + return transcriptions.map((transcription: TranscriptionNotification) => { + return { + state: transcription.state, + mode: transcription.mode, + initialised_at: convertDateNumToDate(transcription.initialised_at), + started_at: convertDateNumToDate(transcription.started_at), + stopped_at: convertDateNumToDate(transcription.stopped_at), + updated_at: convertDateNumToDate(transcription.updated_at), + error: this.toSdkError(transcription?.error), + }; + }); + } private isRecordingRunning(state?: HMSRecordingState): boolean { if (!state) { return false; @@ -149,11 +172,24 @@ export class RoomUpdateManager { if (!notification?.variants) { return hls; } - notification.variants.forEach((_: HLSVariant, index: number) => { - hls.variants.push({ - initialisedAt: convertDateNumToDate(notification?.variants?.[index].initialised_at), - url: '', - }); + notification.variants.forEach((variant: HLSVariant, index: number) => { + if (variant.state !== HMSStreamingState.INITIALISED) { + hls.variants.push({ + meetingURL: variant?.meetingURL, + url: variant?.url, + metadata: variant?.metadata, + playlist_type: variant?.playlist_type, + startedAt: convertDateNumToDate(notification?.variants?.[index].started_at), + initialisedAt: convertDateNumToDate(notification?.variants?.[index].initialised_at), + state: variant.state, + stream_type: variant?.stream_type, + }); + } else { + hls.variants.push({ + initialisedAt: convertDateNumToDate(notification?.variants?.[index].initialised_at), + url: '', + }); + } }); return hls; } @@ -161,7 +197,7 @@ export class RoomUpdateManager { const room = this.store.getRoom(); const running = notification.variants && notification.variants.length > 0 - ? this.isStreamingRunning(notification.variants[0].state) + ? notification.variants.some(variant => this.isStreamingRunning(variant.state)) : false; if (!room) { HMSLogger.w(this.TAG, 'on hls - room not present'); @@ -172,10 +208,20 @@ export class RoomUpdateManager { this.listener?.onRoomUpdate(HMSRoomUpdate.HLS_STREAMING_STATE_UPDATED, room); } + private handleTranscriptionStatus(notification: TranscriptionNotification[]) { + const room = this.store.getRoom(); + if (!room) { + HMSLogger.w(this.TAG, 'on transcription - room not present'); + return; + } + room.transcriptions = this.addTranscriptionDetail(notification) || []; + this.listener?.onRoomUpdate(HMSRoomUpdate.TRANSCRIPTION_STATE_UPDATED, room); + } private convertHls(hlsNotification?: HLSNotification) { + // only checking for zeroth variant intialized const isInitialised = hlsNotification?.variants && hlsNotification.variants.length > 0 - ? hlsNotification.variants[0].state === HMSStreamingState.INITIALISED + ? hlsNotification.variants.some(variant => variant.state === HMSStreamingState.INITIALISED) : false; // handling for initialized state if (isInitialised) { @@ -191,9 +237,11 @@ export class RoomUpdateManager { meetingURL: variant?.meeting_url, url: variant?.url, metadata: variant?.metadata, + playlist_type: variant?.playlist_type, startedAt: convertDateNumToDate(variant?.started_at), initialisedAt: convertDateNumToDate(variant?.initialised_at), state: variant.state, + stream_type: variant?.stream_type, }); }); return hls; diff --git a/packages/hms-video-store/src/notification-manager/managers/TrackManager.ts b/packages/hms-video-store/src/notification-manager/managers/TrackManager.ts index f84d7a2b5a..d3feeba285 100644 --- a/packages/hms-video-store/src/notification-manager/managers/TrackManager.ts +++ b/packages/hms-video-store/src/notification-manager/managers/TrackManager.ts @@ -56,12 +56,11 @@ export class TrackManager { }; handleTrackRemovedPermanently = (notification: TrackStateNotification) => { - HMSLogger.d(this.TAG, `ONTRACKREMOVE`, notification); + HMSLogger.d(this.TAG, `ONTRACKREMOVE permanently`, notification); const trackIds = Object.keys(notification.tracks); trackIds.forEach(trackId => { const trackStateEntry = this.store.getTrackState(trackId); - if (!trackStateEntry) { return; } diff --git a/packages/hms-video-store/src/notification-manager/managers/WhiteboardManager.ts b/packages/hms-video-store/src/notification-manager/managers/WhiteboardManager.ts index fca271ee16..48a0a2d542 100644 --- a/packages/hms-video-store/src/notification-manager/managers/WhiteboardManager.ts +++ b/packages/hms-video-store/src/notification-manager/managers/WhiteboardManager.ts @@ -1,6 +1,7 @@ import { HMSUpdateListener, HMSWhiteboard } from '../../interfaces'; import { Store } from '../../sdk/store'; import HMSTransport from '../../transport'; +import { constructWhiteboardURL } from '../../utils/whiteboard'; import { HMSNotificationMethod } from '../HMSNotificationMethod'; import { WhiteboardInfo } from '../HMSNotifications'; @@ -31,11 +32,20 @@ export class WhiteboardManager { whiteboard.open = isOwner ? prev?.open : open; whiteboard.owner = whiteboard.open ? notification.owner : undefined; - if (!isOwner && whiteboard.open) { - const response = await this.transport.signal.getWhiteboard({ id: notification.id }); - whiteboard.token = response.token; - whiteboard.addr = response.addr; - whiteboard.permissions = response.permissions; + if (whiteboard.open) { + if (isOwner) { + whiteboard.url = prev?.url; + whiteboard.token = prev?.token; + whiteboard.addr = prev?.addr; + whiteboard.permissions = prev?.permissions; + } else { + const response = await this.transport.signal.getWhiteboard({ id: notification.id }); + whiteboard.url = constructWhiteboardURL(response.token, response.addr, this.store.getEnv()); + whiteboard.token = response.token; + whiteboard.addr = response.addr; + whiteboard.permissions = response.permissions; + whiteboard.open = response.permissions.length > 0; + } } this.store.setWhiteboard(whiteboard); diff --git a/packages/hms-video-store/src/notification-manager/managers/onDemandTrackManager.ts b/packages/hms-video-store/src/notification-manager/managers/onDemandTrackManager.ts index 5995d5ca21..d501201a53 100644 --- a/packages/hms-video-store/src/notification-manager/managers/onDemandTrackManager.ts +++ b/packages/hms-video-store/src/notification-manager/managers/onDemandTrackManager.ts @@ -67,7 +67,12 @@ export class OnDemandTrackManager extends TrackManager { const remoteStream = new HMSRemoteStream(new MediaStream(), this.transport.getSubscribeConnection()!); const emptyTrack = LocalTrackManager.getEmptyVideoTrack(); emptyTrack.enabled = !trackInfo.mute; - const track = new HMSRemoteVideoTrack(remoteStream, emptyTrack, trackInfo.source); + const track = new HMSRemoteVideoTrack( + remoteStream, + emptyTrack, + trackInfo.source, + this.store.getRoom()?.disableNoneLayerRequest, + ); track.setTrackId(trackInfo.track_id); track.peerId = hmsPeer.peerId; track.logIdentifier = hmsPeer.name; diff --git a/packages/hms-video-store/src/notification-manager/managers/utils.ts b/packages/hms-video-store/src/notification-manager/managers/utils.ts index 57e3d3aeab..e2a98004e9 100644 --- a/packages/hms-video-store/src/notification-manager/managers/utils.ts +++ b/packages/hms-video-store/src/notification-manager/managers/utils.ts @@ -10,5 +10,6 @@ export const createRemotePeer = (notifPeer: PeerNotificationInfo, store: Store) customerUserId: notifPeer.info.user_id, metadata: notifPeer.info.data, groups: notifPeer.groups, + type: notifPeer.info.type, }); }; diff --git a/packages/hms-video-store/src/notification-manager/notification-manager.test.ts b/packages/hms-video-store/src/notification-manager/notification-manager.test.ts index 985352a714..8b33228517 100644 --- a/packages/hms-video-store/src/notification-manager/notification-manager.test.ts +++ b/packages/hms-video-store/src/notification-manager/notification-manager.test.ts @@ -3,6 +3,7 @@ import { HMSNotificationMethod } from './HMSNotificationMethod'; import { NotificationManager } from './NotificationManager'; import { AnalyticsEventsService } from '../analytics/AnalyticsEventsService'; import { AnalyticsTimer } from '../analytics/AnalyticsTimer'; +import { PluginUsageTracker } from '../common/PluginUsageTracker'; import { DeviceManager } from '../device-manager'; import { EventBus } from '../events/EventBus'; import { HMSAudioListener, HMSPeerUpdate, HMSRoomUpdate, HMSUpdateListener } from '../interfaces'; @@ -10,6 +11,7 @@ import HMSRoom from '../sdk/models/HMSRoom'; import { HMSRemotePeer } from '../sdk/models/peer'; import { Store } from '../sdk/store'; import HMSTransport from '../transport'; +import ITransportObserver from '../transport/ITransportObserver'; let joinHandler: jest.Mock; let previewHandler: jest.Mock; @@ -36,6 +38,9 @@ const store: Store = new Store(); let notificationManager: NotificationManager; let eventBus: EventBus; let transport: HMSTransport; +let deviceManager: DeviceManager; +let analyticsTimer: AnalyticsTimer; +let observer: ITransportObserver; beforeEach(() => { joinHandler = jest.fn(); @@ -57,6 +62,16 @@ beforeEach(() => { pollsUpdateHandler = jest.fn(); whiteboardUpdateHandler = jest.fn(); eventBus = new EventBus(); + deviceManager = new DeviceManager(store, eventBus); + analyticsTimer = new AnalyticsTimer(); + observer = { + onNotification: jest.fn(), + onTrackAdd: jest.fn(), + onTrackRemove: jest.fn(), + onFailure: jest.fn(), + onStateChange: jest.fn(), + onConnected: jest.fn(), + }; const mockMediaStream = { id: 'native-stream-id', getVideoTracks: jest.fn(() => [ @@ -82,19 +97,13 @@ beforeEach(() => { global.HTMLCanvasElement.prototype.captureStream = jest.fn().mockImplementation(() => mockMediaStream); transport = new HMSTransport( - { - onNotification: jest.fn(), - onTrackAdd: jest.fn(), - onTrackRemove: jest.fn(), - onFailure: jest.fn(), - onStateChange: jest.fn(), - onConnected: jest.fn(), - }, - new DeviceManager(store, eventBus), + observer, + deviceManager, store, eventBus, new AnalyticsEventsService(store), - new AnalyticsTimer(), + analyticsTimer, + new PluginUsageTracker(eventBus), ); store.setRoom(new HMSRoom('1234')); @@ -118,6 +127,8 @@ beforeEach(() => { onWhiteboardUpdate: whiteboardUpdateHandler, }; + transport.setListener(listener); + audioListener = { onAudioLevelUpdate: audioUpdateHandler }; notificationManager = new NotificationManager(store, eventBus, transport, listener, audioListener); diff --git a/packages/hms-video-store/src/plugins/audio/AudioPluginsAnalytics.ts b/packages/hms-video-store/src/plugins/audio/AudioPluginsAnalytics.ts index 08801d07eb..30629c4bdf 100644 --- a/packages/hms-video-store/src/plugins/audio/AudioPluginsAnalytics.ts +++ b/packages/hms-video-store/src/plugins/audio/AudioPluginsAnalytics.ts @@ -24,6 +24,7 @@ export class AudioPluginsAnalytics { this.addedTimestamps[name] = Date.now(); this.initTime[name] = 0; this.pluginSampleRate[name] = sampleRate; + this.eventBus.analytics.publish(MediaPluginsAnalyticsFactory.added(name, this.addedTimestamps[name])); } removed(name: string) { diff --git a/packages/hms-video-store/src/plugins/audio/HMSAudioPluginsManager.ts b/packages/hms-video-store/src/plugins/audio/HMSAudioPluginsManager.ts index 9c4d4d96a1..fa84d76ed4 100644 --- a/packages/hms-video-store/src/plugins/audio/HMSAudioPluginsManager.ts +++ b/packages/hms-video-store/src/plugins/audio/HMSAudioPluginsManager.ts @@ -1,9 +1,13 @@ import { AudioPluginsAnalytics } from './AudioPluginsAnalytics'; import { HMSAudioPlugin, HMSPluginUnsupportedTypes } from './HMSAudioPlugin'; //HMSAudioPluginType +import AnalyticsEventFactory from '../../analytics/AnalyticsEventFactory'; import { ErrorFactory } from '../../error/ErrorFactory'; import { HMSAction } from '../../error/HMSAction'; import { EventBus } from '../../events/EventBus'; +import { HMSAudioTrackSettingsBuilder } from '../../media/settings'; +import { standardMediaConstraints } from '../../media/settings/constants'; import { HMSLocalAudioTrack } from '../../media/tracks'; +import Room from '../../sdk/models/HMSRoom'; import HMSLogger from '../../utils/logger'; const DEFAULT_SAMPLE_RATE = 48000; @@ -30,7 +34,7 @@ export class HMSAudioPluginsManager { private readonly TAG = '[AudioPluginsManager]'; private readonly hmsTrack: HMSLocalAudioTrack; // Map maintains the insertion order - private readonly pluginsMap: Map; + readonly pluginsMap: Map; private audioContext?: AudioContext; private sourceNode?: MediaStreamAudioSourceNode; @@ -40,12 +44,14 @@ export class HMSAudioPluginsManager { // This will replace the native track in peer connection when plugins are enabled private outputTrack?: MediaStreamTrack; private pluginAddInProgress = false; + private room?: Room; - constructor(track: HMSLocalAudioTrack, eventBus: EventBus) { + constructor(track: HMSLocalAudioTrack, private eventBus: EventBus, room?: Room) { this.hmsTrack = track; this.pluginsMap = new Map(); this.analytics = new AudioPluginsAnalytics(eventBus); this.createAudioContext(); + this.room = room; } getPlugins(): string[] { @@ -69,6 +75,38 @@ export class HMSAudioPluginsManager { throw err; } + switch (plugin.getName()) { + case 'HMSKrispPlugin': { + if (!this.room?.isNoiseCancellationEnabled) { + const errorMessage = 'Krisp Noise Cancellation is not enabled for this room'; + if (this.pluginsMap.size === 0) { + throw Error(errorMessage); + } else { + HMSLogger.w(this.TAG, errorMessage); + return; + } + } + this.eventBus.analytics.publish(AnalyticsEventFactory.krispStart()); + const { settings } = this.hmsTrack; + const newAudioTrackSettings = new HMSAudioTrackSettingsBuilder() + .codec(settings.codec) + .maxBitrate(settings.maxBitrate) + .deviceId(settings.deviceId!) + .advanced([ + ...standardMediaConstraints, + // @ts-ignore + { autoGainControl: { exact: false } }, + // @ts-ignore + { noiseSuppression: { exact: false } }, + ]) + .audioMode(settings.audioMode) + .build(); + await this.hmsTrack.setSettings(newAudioTrackSettings); + break; + } + + default: + } this.pluginAddInProgress = true; try { @@ -87,6 +125,8 @@ export class HMSAudioPluginsManager { } await this.validateAndThrow(name, plugin); + // @ts-ignore + plugin.setEventBus?.(this.eventBus); try { if (this.pluginsMap.size === 0) { @@ -100,6 +140,7 @@ export class HMSAudioPluginsManager { this.pluginsMap.set(name, plugin); await this.processPlugin(plugin); await this.connectToDestination(); + await this.updateProcessedTrack(); } catch (err) { HMSLogger.e(this.TAG, 'failed to add plugin', err); throw err; @@ -138,6 +179,29 @@ export class HMSAudioPluginsManager { } async removePlugin(plugin: HMSAudioPlugin) { + switch (plugin.getName()) { + case 'HMSKrispPlugin': { + this.eventBus.analytics.publish(AnalyticsEventFactory.krispStop()); + const { settings } = this.hmsTrack; + const newAudioTrackSettings = new HMSAudioTrackSettingsBuilder() + .codec(settings.codec) + .maxBitrate(settings.maxBitrate) + .deviceId(settings.deviceId!) + .advanced([ + ...standardMediaConstraints, + // @ts-ignore + { autoGainControl: { exact: true } }, + // @ts-ignore + { noiseSuppression: { exact: true } }, + ]) + .audioMode(settings.audioMode) + .build(); + await this.hmsTrack.setSettings(newAudioTrackSettings); + break; + } + default: + break; + } await this.removePluginInternal(plugin); if (this.pluginsMap.size === 0) { // remove all previous nodes @@ -184,6 +248,7 @@ export class HMSAudioPluginsManager { for (const plugin of plugins) { await this.addPlugin(plugin); } + this.updateProcessedTrack(); } private async initAudioNodes() { @@ -195,16 +260,19 @@ export class HMSAudioPluginsManager { if (!this.destinationNode) { this.destinationNode = this.audioContext.createMediaStreamDestination(); this.outputTrack = this.destinationNode.stream.getAudioTracks()[0]; - try { - await this.hmsTrack.setProcessedTrack(this.outputTrack); - } catch (err) { - HMSLogger.e(this.TAG, 'error in setting processed track', err); - throw err; - } } } } + private async updateProcessedTrack() { + try { + await this.hmsTrack.setProcessedTrack(this.outputTrack); + } catch (err) { + HMSLogger.e(this.TAG, 'error in setting processed track', err); + throw err; + } + } + private async processPlugin(plugin: HMSAudioPlugin) { try { const currentNode = await plugin.processAudioTrack( diff --git a/packages/hms-video-store/src/plugins/video/HMSMediaStreamPluginsManager.ts b/packages/hms-video-store/src/plugins/video/HMSMediaStreamPluginsManager.ts index 332badd326..5bc32bcfcf 100644 --- a/packages/hms-video-store/src/plugins/video/HMSMediaStreamPluginsManager.ts +++ b/packages/hms-video-store/src/plugins/video/HMSMediaStreamPluginsManager.ts @@ -2,19 +2,39 @@ import { HMSMediaStreamPlugin } from './HMSMediaStreamPlugin'; import { VideoPluginsAnalytics } from './VideoPluginsAnalytics'; import { EventBus } from '../../events/EventBus'; import { HMSException } from '../../internal'; +import Room from '../../sdk/models/HMSRoom'; import HMSLogger from '../../utils/logger'; export class HMSMediaStreamPluginsManager { + private readonly TAG = '[MediaStreamPluginsManager]'; private analytics: VideoPluginsAnalytics; - private plugins: Set; + readonly plugins: Set; + private room?: Room; - constructor(eventBus: EventBus) { + constructor(eventBus: EventBus, room?: Room) { this.plugins = new Set(); this.analytics = new VideoPluginsAnalytics(eventBus); + this.room = room; } addPlugins(plugins: HMSMediaStreamPlugin[]): void { - plugins.forEach(plugin => this.plugins.add(plugin)); + plugins.forEach(plugin => { + switch (plugin.getName()) { + case 'HMSEffectsPlugin': + if (!this.room?.isEffectsEnabled) { + const errorMessage = 'Effects Virtual Background is not enabled for this room'; + if (this.plugins.size === 0) { + throw Error(errorMessage); + } else { + HMSLogger.w(this.TAG, errorMessage); + return; + } + } + break; + default: + } + this.plugins.add(plugin); + }); } removePlugins(plugins: HMSMediaStreamPlugin[]) { @@ -43,4 +63,8 @@ export class HMSMediaStreamPluginsManager { getPlugins(): string[] { return Array.from(this.plugins).map(plugin => plugin.getName()); } + + async cleanup() { + this.removePlugins(Array.from(this.plugins)); + } } diff --git a/packages/hms-video-store/src/plugins/video/HMSVideoPluginsManager.ts b/packages/hms-video-store/src/plugins/video/HMSVideoPluginsManager.ts index 15ffd20012..2a018cbfce 100644 --- a/packages/hms-video-store/src/plugins/video/HMSVideoPluginsManager.ts +++ b/packages/hms-video-store/src/plugins/video/HMSVideoPluginsManager.ts @@ -6,7 +6,7 @@ import { HMSAction } from '../../error/HMSAction'; import { EventBus } from '../../events/EventBus'; import { HMSLocalVideoTrack } from '../../media/tracks'; import HMSLogger from '../../utils/logger'; -import { workerSleep } from '../../utils/timer-utils'; +import { reusableWorker, workerSleep } from '../../utils/timer-utils'; import { HMSPluginUnsupportedTypes } from '../audio'; const DEFAULT_FRAME_RATE = 24; @@ -47,7 +47,7 @@ export class HMSVideoPluginsManager { private pluginsLoopRunning = false; private pluginsLoopState: 'paused' | 'running' = 'paused'; private readonly hmsTrack: HMSLocalVideoTrack; - private readonly pluginsMap: Map; // plugin names to their instance mapping + readonly pluginsMap: Map; // plugin names to their instance mapping private inputVideo?: HTMLVideoElement; private inputCanvas?: CanvasElement; private outputCanvas?: CanvasElement; @@ -57,6 +57,7 @@ export class HMSVideoPluginsManager { private pluginNumFramesToSkip: Record; private pluginNumFramesSkipped: Record; private canvases: Array; //array of canvases to store intermediate result + private reusableWorker = reusableWorker(); constructor(track: HMSLocalVideoTrack, eventBus: EventBus) { this.hmsTrack = track; @@ -288,7 +289,7 @@ export class HMSVideoPluginsManager { this.resetCanvases(); } this.pluginsLoopState = 'paused'; - await workerSleep(sleepTimeMs); + await this.reusableWorker.sleep(sleepTimeMs); continue; } let processingTime = 0; @@ -306,7 +307,7 @@ export class HMSVideoPluginsManager { } this.pluginsLoopState = 'running'; // take into account processing time to decide time to wait for the next loop - await workerSleep(sleepTimeMs - processingTime); + await this.reusableWorker.sleep(sleepTimeMs - processingTime); } } diff --git a/packages/hms-video-store/src/reactive-store/HMSNotifications.ts b/packages/hms-video-store/src/reactive-store/HMSNotifications.ts index 8c9b48db8c..b0d5c919d5 100644 --- a/packages/hms-video-store/src/reactive-store/HMSNotifications.ts +++ b/packages/hms-video-store/src/reactive-store/HMSNotifications.ts @@ -1,5 +1,10 @@ import { EventEmitter2 as EventEmitter } from 'eventemitter2'; -import { PEER_NOTIFICATION_TYPES, POLL_NOTIFICATION_TYPES, TRACK_NOTIFICATION_TYPES } from './common/mapping'; +import { + PEER_NOTIFICATION_TYPES, + POLL_NOTIFICATION_TYPES, + TRACK_NOTIFICATION_TYPES, + TRANSCRIPTION_NOTIFICATION_TYPES, +} from './common/mapping'; import { IHMSStore } from '../IHMSStore'; import * as sdkTypes from '../internal'; import { @@ -172,6 +177,14 @@ export class HMSNotifications | sdkTypes.HMSPoll + | sdkTypes.HMSTranscriptionInfo[] | null, severity?: HMSNotificationSeverity, message = '', diff --git a/packages/hms-video-store/src/reactive-store/HMSReactiveStore.ts b/packages/hms-video-store/src/reactive-store/HMSReactiveStore.ts index 1a265901c0..c6b30175c3 100644 --- a/packages/hms-video-store/src/reactive-store/HMSReactiveStore.ts +++ b/packages/hms-video-store/src/reactive-store/HMSReactiveStore.ts @@ -13,6 +13,7 @@ import { HMSNotifications } from './HMSNotifications'; import { HMSSDKActions } from './HMSSDKActions'; import { NamedSetState } from './internalTypes'; import { storeNameWithTabTitle } from '../common/storeName'; +import { HMSDiagnosticsInterface } from '../diagnostics/interfaces'; import { IHMSActions } from '../IHMSActions'; import { IHMSStatsStoreReadOnly, IHMSStore, IHMSStoreReadOnly, IStore } from '../IHMSStore'; import { isBrowser } from '../internal'; @@ -34,6 +35,7 @@ export class HMSReactiveStore; private readonly notifications: HMSNotifications; private stats?: HMSStats; + private diagnostics?: HMSDiagnosticsInterface; /** @TODO store flag for both HMSStore and HMSInternalsStore */ private initialTriggerOnSubscribe: boolean; @@ -133,6 +135,14 @@ export class HMSReactiveStore { + if (!this.diagnostics) { + this.diagnostics = this.actions.initDiagnostics(); + } + + return this.diagnostics; + }; + /** * @internal */ diff --git a/packages/hms-video-store/src/reactive-store/HMSSDKActions.ts b/packages/hms-video-store/src/reactive-store/HMSSDKActions.ts index bc53136543..9b880b70f7 100644 --- a/packages/hms-video-store/src/reactive-store/HMSSDKActions.ts +++ b/packages/hms-video-store/src/reactive-store/HMSSDKActions.ts @@ -16,9 +16,12 @@ import { HMSSessionStore } from './HMSSessionStore'; import { NamedSetState } from './internalTypes'; import { HMSLogger } from '../common/ui-logger'; import { BeamSpeakerLabelsLogger } from '../controller/beam/BeamSpeakerLabelsLogger'; +import { Diagnostics } from '../diagnostics'; +import { HMSSessionFeedback } from '../end-call-feedback'; import { IHMSActions } from '../IHMSActions'; import { IHMSStore } from '../IHMSStore'; import { + HMSAudioMode, HMSAudioPlugin, HMSAudioTrack as SDKHMSAudioTrack, HMSChangeMultiTrackStateParams as SDKHMSChangeMultiTrackStateParams, @@ -88,8 +91,8 @@ import { selectTracksMap, selectVideoTrackByID, } from '../selectors'; - -// import { ActionBatcher } from './sdkUtils/ActionBatcher'; +import { FindPeerByNameRequestParams } from '../signal/interfaces'; +import { HMSStats } from '../webrtc-stats'; /** * This class implements the IHMSActions interface for 100ms SDK. It connects with SDK @@ -133,6 +136,9 @@ export class HMSSDKActions(this.sdk, this.setSessionStoreValueLocally.bind(this)); this.actionBatcher = new ActionBatcher(store); } + submitSessionFeedback(feedback: HMSSessionFeedback, eventEndpoint?: string): Promise { + return this.sdk.submitSessionFeedback(feedback, eventEndpoint); + } getLocalTrack(trackID: string) { return this.sdk.store.getLocalPeerTracks().find(track => track.trackId === trackID); @@ -360,6 +366,7 @@ export class HMSSDKActions) { const trackID = this.store.getState(selectLocalAudioTrackID); + if (trackID) { await this.setSDKLocalAudioTrackSettings(trackID, settings); this.syncRoomState('setAudioSettings'); @@ -415,6 +422,10 @@ export class HMSSDKActions { + store.messages.byID[hmsMessage.id] = hmsMessage; + store.messages.allIDs.push(hmsMessage.id); + }, 'newMessage'); } setMessageRead(readStatus: boolean, messageId?: string) { @@ -594,6 +609,19 @@ export class HMSSDKActions SDKToHMS.convertPeer(peer) as HMSPeer) }; + } + getPeerListIterator(options?: HMSPeerListIteratorOptions) { const iterator = this.sdk.getPeerListIterator(options); return { @@ -687,6 +715,14 @@ export class HMSSDKActions { + await this.sdk.stopTranscription(params); + } + async sendHLSTimedMetadata(metadataList: sdkTypes.HLSTimedMetadata[]): Promise { await this.sdk.sendHLSTimedMetadata(metadataList); } @@ -771,6 +807,36 @@ export class HMSSDKActions { Object.assign(store.room, SDKToHMS.convertRoom(room, this.sdk.getLocalPeer()?.peerId)); }, type); + if (type === sdkTypes.HMSRoomUpdate.TRANSCRIPTION_STATE_UPDATED) { + this.hmsNotifications.sendTranscriptionUpdate(room.transcriptions); + } } protected onPeerUpdate(type: sdkTypes.HMSPeerUpdate, sdkPeer: sdkTypes.HMSPeer | sdkTypes.HMSPeer[]) { @@ -1136,6 +1217,9 @@ export class HMSSDKActions> = (fn, name) => { return this.store.namedSetState(fn, name); }; + + /** + * @internal + * This will be used by beam to check if the recording should continue, it will pass __hms.stats + * It will poll at a fixed interval and start an exit timer if the method fails twice consecutively + * The exit timer is stopped if the method returns true before that + * @param hmsStats + */ + hasActiveElements(hmsStats: HMSStats): boolean { + const isWhiteboardPresent = Object.keys(this.store.getState().whiteboards).length > 0; + const isQuizOrPollPresent = Object.keys(this.store.getState().polls).length > 0; + const peerCount = Object.keys(this.store.getState().peers).length > 0; + const remoteTracks = hmsStats.getState().remoteTrackStats; + return ( + peerCount && + (isWhiteboardPresent || + isQuizOrPollPresent || + Object.values(remoteTracks).some(track => track && typeof track.bitrate === 'number' && track.bitrate > 0)) + ); + } } diff --git a/packages/hms-video-store/src/reactive-store/adapter.ts b/packages/hms-video-store/src/reactive-store/adapter.ts index aff7e202fb..b6c4ea7b29 100644 --- a/packages/hms-video-store/src/reactive-store/adapter.ts +++ b/packages/hms-video-store/src/reactive-store/adapter.ts @@ -24,6 +24,7 @@ import { HMSRoom, HMSScreenVideoTrack, HMSTrack, + HMSTrackException, HMSTrackFacingMode, HMSVideoTrack, } from '../schema'; @@ -51,6 +52,7 @@ export class SDKToHMS { joinedAt: sdkPeer.joinedAt, groups: sdkPeer.groups, isHandRaised: sdkPeer.isHandRaised, + type: sdkPeer.type, }; } @@ -138,10 +140,11 @@ export class SDKToHMS { } static convertRoom(sdkRoom: sdkTypes.HMSRoom, sdkLocalPeerId?: string): Partial { - const { recording, rtmp, hls } = SDKToHMS.convertRecordingStreamingState( - sdkRoom?.recording, - sdkRoom?.rtmp, - sdkRoom?.hls, + const { recording, rtmp, hls, transcriptions } = SDKToHMS.convertRecordingStreamingState( + sdkRoom.recording, + sdkRoom.rtmp, + sdkRoom.hls, + sdkRoom.transcriptions, ); return { id: sdkRoom.id, @@ -150,13 +153,18 @@ export class SDKToHMS { recording, rtmp, hls, + transcriptions, sessionId: sdkRoom.sessionId, startedAt: sdkRoom.startedAt, joinedAt: sdkRoom.joinedAt, peerCount: sdkRoom.peerCount, isLargeRoom: sdkRoom.large_room_optimization, isEffectsEnabled: sdkRoom.isEffectsEnabled, + disableNoneLayerRequest: sdkRoom.disableNoneLayerRequest, + isVBEnabled: sdkRoom.isVBEnabled, effectsKey: sdkRoom.effectsKey, + isHipaaEnabled: sdkRoom.isHipaaEnabled, + isNoiseCancellationEnabled: sdkRoom.isNoiseCancellationEnabled, }; } @@ -196,8 +204,9 @@ export class SDKToHMS { }; } - static convertException(sdkException: sdkTypes.HMSException): HMSException { - return { + static convertException(sdkException: sdkTypes.HMSException): HMSException | HMSTrackException { + const isTrackException = 'trackType' in sdkException; + const exp = { code: sdkException.code, action: sdkException.action, name: sdkException.name, @@ -206,7 +215,12 @@ export class SDKToHMS { isTerminal: sdkException.isTerminal, nativeError: sdkException.nativeError, timestamp: new Date(), - }; + } as HMSException; + if (isTrackException) { + (exp as HMSTrackException).trackType = (sdkException as sdkTypes.HMSTrackException)?.trackType; + return exp as HMSTrackException; + } + return exp; } static convertDeviceChangeUpdate(sdkDeviceChangeEvent: sdkTypes.HMSDeviceChangeEvent): HMSDeviceChangeEvent { @@ -273,7 +287,13 @@ export class SDKToHMS { recording?: sdkTypes.HMSRecording, rtmp?: sdkTypes.HMSRTMP, hls?: sdkTypes.HMSHLS, - ): { recording: sdkTypes.HMSRecording; rtmp: sdkTypes.HMSRTMP; hls: sdkTypes.HMSHLS } { + transcriptions?: sdkTypes.HMSTranscriptionInfo[], + ): { + recording: sdkTypes.HMSRecording; + rtmp: sdkTypes.HMSRTMP; + hls: sdkTypes.HMSHLS; + transcriptions: sdkTypes.HMSTranscriptionInfo[]; + } { return { recording: { browser: { @@ -292,6 +312,7 @@ export class SDKToHMS { running: !!hls?.running, error: hls?.error, }, + transcriptions: transcriptions || [], }; } } diff --git a/packages/hms-video-store/src/reactive-store/common/mapping.ts b/packages/hms-video-store/src/reactive-store/common/mapping.ts index 9cb9bc4e29..e4fa809056 100644 --- a/packages/hms-video-store/src/reactive-store/common/mapping.ts +++ b/packages/hms-video-store/src/reactive-store/common/mapping.ts @@ -31,5 +31,12 @@ export const POLL_NOTIFICATION_TYPES: PollNotificationMap = { [sdkTypes.HMSPollsUpdate.POLL_STOPPED]: HMSNotificationTypes.POLL_STOPPED, [sdkTypes.HMSPollsUpdate.POLL_STATS_UPDATED]: HMSNotificationTypes.POLL_VOTES_UPDATED, [sdkTypes.HMSPollsUpdate.POLLS_LIST]: HMSNotificationTypes.POLLS_LIST, - // [sdkTypes.HMSPollsUpdate.POLL_LEADERBOARD_SHARED]: HMSNotificationTypes.POLL_LEADERBOARD_SHARED, +}; + +type TranscriptionNotificationMap = { + [key in sdkTypes.HMSRoomUpdate.TRANSCRIPTION_STATE_UPDATED]: HMSNotificationTypes; +}; + +export const TRANSCRIPTION_NOTIFICATION_TYPES: TranscriptionNotificationMap = { + [sdkTypes.HMSRoomUpdate.TRANSCRIPTION_STATE_UPDATED]: HMSNotificationTypes.TRANSCRIPTION_STATE_UPDATED, }; diff --git a/packages/hms-video-store/src/rtc-stats/HMSWebrtcInternals.ts b/packages/hms-video-store/src/rtc-stats/HMSWebrtcInternals.ts index 0873ab4d8d..897b530b68 100644 --- a/packages/hms-video-store/src/rtc-stats/HMSWebrtcInternals.ts +++ b/packages/hms-video-store/src/rtc-stats/HMSWebrtcInternals.ts @@ -14,23 +14,18 @@ export class HMSWebrtcInternals { private isMonitored = false; private hmsStats?: HMSWebrtcStats; - constructor( - private readonly store: Store, - private readonly eventBus: EventBus, - private publishConnection?: RTCPeerConnection, - private subscribeConnection?: RTCPeerConnection, - ) {} + constructor(private readonly store: Store, private readonly eventBus: EventBus) {} - getPublishPeerConnection() { - return this.publishConnection; + getCurrentStats() { + return this.hmsStats; } - getSubscribePeerConnection() { - return this.subscribeConnection; + getPublishPeerConnection() { + return this.hmsStats?.getPublishPeerConnection(); } - getCurrentStats() { - return this.hmsStats; + getSubscribePeerConnection() { + return this.hmsStats?.getSubscribePeerConnection(); } onStatsChange(statsChangeCb: (stats: HMSWebrtcStats) => void) { @@ -42,7 +37,9 @@ export class HMSWebrtcInternals { private handleStatsUpdate = async () => { await this.hmsStats?.updateStats(); - this.eventBus.statsUpdate.publish(this.hmsStats); + if (this.hmsStats) { + this.eventBus.statsUpdate.publish(this.hmsStats); + } }; /** @@ -50,17 +47,11 @@ export class HMSWebrtcInternals { * @internal */ setPeerConnections({ publish, subscribe }: { publish?: RTCPeerConnection; subscribe?: RTCPeerConnection }) { - this.publishConnection = publish; - this.subscribeConnection = subscribe; - - this.hmsStats = new HMSWebrtcStats( - { - publish: this.publishConnection?.getStats.bind(this.publishConnection), - subscribe: this.subscribeConnection?.getStats.bind(this.subscribeConnection), - }, - this.store, - this.eventBus, - ); + if (this.hmsStats) { + this.hmsStats.setPeerConnections({ publish, subscribe }); + } else { + this.hmsStats = new HMSWebrtcStats(this.store, this.eventBus, publish, subscribe); + } } /** diff --git a/packages/hms-video-store/src/rtc-stats/HMSWebrtcStats.ts b/packages/hms-video-store/src/rtc-stats/HMSWebrtcStats.ts index 9090fd69df..33ea367506 100644 --- a/packages/hms-video-store/src/rtc-stats/HMSWebrtcStats.ts +++ b/packages/hms-video-store/src/rtc-stats/HMSWebrtcStats.ts @@ -10,7 +10,7 @@ import AnalyticsEventFactory from '../analytics/AnalyticsEventFactory'; import { ErrorFactory } from '../error/ErrorFactory'; import { HMSAction } from '../error/HMSAction'; import { EventBus } from '../events/EventBus'; -import { HMSPeerStats, HMSTrackStats, PeerConnectionType } from '../interfaces/webrtc-stats'; +import { HMSPeerStats, HMSTrackStats } from '../interfaces/webrtc-stats'; import { HMSLocalTrack, HMSRemoteAudioTrack, HMSRemoteTrack, HMSRemoteVideoTrack } from '../media/tracks'; import { Store } from '../sdk/store'; import HMSLogger from '../utils/logger'; @@ -27,13 +27,27 @@ export class HMSWebrtcStats { * this is initialized */ constructor( - private getStats: Record, private store: Store, private readonly eventBus: EventBus, + private publishConnection?: RTCPeerConnection, + private subscribeConnection?: RTCPeerConnection, ) { this.localPeerID = this.store.getLocalPeer()?.peerId; } + setPeerConnections({ publish, subscribe }: { publish?: RTCPeerConnection; subscribe?: RTCPeerConnection }) { + this.publishConnection = publish; + this.subscribeConnection = subscribe; + } + + getPublishPeerConnection() { + return this.publishConnection; + } + + getSubscribePeerConnection() { + return this.subscribeConnection; + } + getLocalPeerStats = (): HMSPeerStats | undefined => { return this.peerStats[this.localPeerID!]; }; @@ -63,7 +77,7 @@ export class HMSWebrtcStats { const prevLocalPeerStats = this.getLocalPeerStats(); let publishReport: RTCStatsReport | undefined; try { - publishReport = await this.getStats.publish?.(); + publishReport = await this.publishConnection?.getStats(); } catch (err: any) { this.eventBus.analytics.publish( AnalyticsEventFactory.rtcStatsFailed(ErrorFactory.WebrtcErrors.StatsFailed(HMSAction.PUBLISH, err.message)), @@ -72,10 +86,9 @@ export class HMSWebrtcStats { } const publishStats: HMSPeerStats['publish'] | undefined = publishReport && getLocalPeerStatsFromReport('publish', publishReport, prevLocalPeerStats); - let subscribeReport: RTCStatsReport | undefined; try { - subscribeReport = await this.getStats.subscribe?.(); + subscribeReport = await this.subscribeConnection?.getStats(); } catch (err: any) { this.eventBus.analytics.publish( AnalyticsEventFactory.rtcStatsFailed(ErrorFactory.WebrtcErrors.StatsFailed(HMSAction.SUBSCRIBE, err.message)), diff --git a/packages/hms-video-store/src/rtc-stats/utils.ts b/packages/hms-video-store/src/rtc-stats/utils.ts index 35d63a3f7f..8e6c1a9232 100644 --- a/packages/hms-video-store/src/rtc-stats/utils.ts +++ b/packages/hms-video-store/src/rtc-stats/utils.ts @@ -116,15 +116,15 @@ export const getTrackStats = async ( } return ( - trackStats && - Object.assign(trackStats, { + trackStats && { + ...trackStats, bitrate, packetsLostRate, - peerId: track.peerId, + peerID: track.peerId, enabled: track.enabled, peerName, codec: trackStats.codec, - }) + } ); }; diff --git a/packages/hms-video-store/src/schema/error.ts b/packages/hms-video-store/src/schema/error.ts index 7a19a4a7a0..ca9dcb4572 100644 --- a/packages/hms-video-store/src/schema/error.ts +++ b/packages/hms-video-store/src/schema/error.ts @@ -1,3 +1,5 @@ +import { HMSTrackExceptionTrackType } from '../media/tracks/HMSTrackExceptionTrackType'; + /** * any mid call error notification will be in this format */ @@ -11,3 +13,7 @@ export interface HMSException { timestamp: Date; nativeError?: Error; } + +export interface HMSTrackException extends HMSException { + trackType: HMSTrackExceptionTrackType; +} diff --git a/packages/hms-video-store/src/schema/notification.ts b/packages/hms-video-store/src/schema/notification.ts index b471bdfa91..eb7fe063c5 100644 --- a/packages/hms-video-store/src/schema/notification.ts +++ b/packages/hms-video-store/src/schema/notification.ts @@ -4,7 +4,7 @@ import { HMSMessage } from './message'; import { HMSPeer, HMSTrack } from './peer'; import { HMSPlaylistItem } from './playlist'; import { HMSChangeMultiTrackStateRequest, HMSChangeTrackStateRequest, HMSLeaveRoomRequest } from './requests'; -import { HMSPoll } from '../internal'; +import { HMSPoll, HMSTranscriptionInfo } from '../internal'; interface BaseNotification { id: number; @@ -75,10 +75,13 @@ export interface HMSReconnectionNotification extends BaseNotification { export interface HMSPollNotification extends BaseNotification { type: HMSNotificationTypes.POLL_STARTED | HMSNotificationTypes.POLL_STOPPED | HMSNotificationTypes.POLL_VOTES_UPDATED; - // | HMSNotificationTypes.POLL_LEADERBOARD_SHARED; data: HMSPoll; } +export interface HMSTranscriptionNotification extends BaseNotification { + type: HMSNotificationTypes.TRANSCRIPTION_STATE_UPDATED; + data: HMSTranscriptionInfo[]; +} export type HMSNotification = | HMSPeerNotification | HMSPeerListNotification @@ -90,6 +93,7 @@ export type HMSNotification = | HMSLeaveRoomRequestNotification | HMSDeviceChangeEventNotification | HMSReconnectionNotification + | HMSTranscriptionNotification | HMSPlaylistItemNotification; export enum HMSNotificationSeverity { @@ -126,8 +130,8 @@ export enum HMSNotificationTypes { POLL_STOPPED = 'POLL_STOPPED', POLL_VOTES_UPDATED = 'POLL_VOTES_UPDATED', POLLS_LIST = 'POLLS_LIST', - // POLL_LEADERBOARD_SHARED = 'POLL_LEADERBOARD_SHARED', HAND_RAISE_CHANGED = 'HAND_RAISE_CHANGED', + TRANSCRIPTION_STATE_UPDATED = 'TRANSCRIPTION_STATE_UPDATED', } export type HMSNotificationMapping = { @@ -159,9 +163,9 @@ export type HMSNotificationMapping = { [HMSNotificationTypes.POLL_STOPPED]: HMSPollNotification; [HMSNotificationTypes.POLL_VOTES_UPDATED]: HMSPollNotification; [HMSNotificationTypes.POLLS_LIST]: HMSPollNotification; - // [HMSNotificationTypes.POLL_LEADERBOARD_SHARED]: HMSPollNotification; [HMSNotificationTypes.POLL_CREATED]: HMSPollNotification; [HMSNotificationTypes.HAND_RAISE_CHANGED]: HMSPeerNotification; + [HMSNotificationTypes.TRANSCRIPTION_STATE_UPDATED]: HMSTranscriptionNotification; }[T]; export type MappedNotifications = { diff --git a/packages/hms-video-store/src/schema/peer.ts b/packages/hms-video-store/src/schema/peer.ts index 51b0386519..a9d40d0dfe 100644 --- a/packages/hms-video-store/src/schema/peer.ts +++ b/packages/hms-video-store/src/schema/peer.ts @@ -5,6 +5,7 @@ import { HMSSimulcastLayerDefinition, ScreenCaptureHandle, } from '../interfaces'; +import { HMSPeerType } from '../interfaces/peer/hms-peer'; export type HMSPeerID = string; export type HMSTrackID = string; @@ -42,6 +43,7 @@ export interface HMSPeer { joinedAt?: Date; groups?: HMSGroupName[]; isHandRaised: boolean; + type: HMSPeerType; } /** diff --git a/packages/hms-video-store/src/schema/room.ts b/packages/hms-video-store/src/schema/room.ts index 1acc22a533..b14b2dacf5 100644 --- a/packages/hms-video-store/src/schema/room.ts +++ b/packages/hms-video-store/src/schema/room.ts @@ -1,5 +1,5 @@ import { HMSPeerID } from './peer'; -import { HLSVariant, HMSHLS, HMSRecording, HMSRTMP } from '../interfaces'; +import { HLSVariant, HMSHLS, HMSRecording, HMSRTMP, HMSTranscriptionInfo } from '../interfaces'; export type { HMSRecording, HMSRTMP, HMSHLS, HLSVariant }; export type HMSRoomID = string; @@ -33,11 +33,16 @@ export interface HMSRoom { sessionId: string; startedAt?: Date; joinedAt?: Date; + transcriptions?: HMSTranscriptionInfo[]; /** * if this number is available room.peers is not guaranteed to have all the peers. */ peerCount?: number; isLargeRoom?: boolean; isEffectsEnabled?: boolean; + disableNoneLayerRequest?: boolean; + isVBEnabled?: boolean; effectsKey?: string; + isHipaaEnabled?: boolean; + isNoiseCancellationEnabled?: boolean; } diff --git a/packages/hms-video-store/src/schema/settings.ts b/packages/hms-video-store/src/schema/settings.ts index d4a78130f5..15b6ca261d 100644 --- a/packages/hms-video-store/src/schema/settings.ts +++ b/packages/hms-video-store/src/schema/settings.ts @@ -1,5 +1,14 @@ +import { HMSAudioMode } from '../interfaces'; + export interface HMSMediaSettings { audioInputDeviceId: string; videoInputDeviceId: string; audioOutputDeviceId?: string; + audioMode?: HMSAudioMode; +} + +export interface DebugInfo { + websocketURL?: string; + enabledFlags?: string[]; + initEndpoint?: string; } diff --git a/packages/hms-video-store/src/sdk/LocalTrackManager.test.ts b/packages/hms-video-store/src/sdk/LocalTrackManager.test.ts index 5da120f4cf..e1788ae25e 100644 --- a/packages/hms-video-store/src/sdk/LocalTrackManager.test.ts +++ b/packages/hms-video-store/src/sdk/LocalTrackManager.test.ts @@ -5,7 +5,7 @@ import { AnalyticsTimer } from '../analytics/AnalyticsTimer'; import { DeviceManager } from '../device-manager'; import { HMSException } from '../error/HMSException'; import { EventBus } from '../events/EventBus'; -import { HMSLocalVideoTrack, HMSTrackType } from '../internal'; +import { HMSLocalVideoTrack, HMSPeerType, HMSTrackType } from '../internal'; import { HMSLocalStream } from '../media/streams/HMSLocalStream'; import { HMSTrack } from '../media/tracks'; import { PolicyParams } from '../notification-manager'; @@ -30,6 +30,7 @@ const testObserver: ITransportObserver = { let testStore = new Store(); let testEventBus = new EventBus(); +let analyticsTimer = new AnalyticsTimer(); const policyParams: PolicyParams = { name: 'host', @@ -96,6 +97,7 @@ const publishParams = hostRole.publishParams; let localPeer = new HMSLocalPeer({ name: 'test', role: hostRole, + type: HMSPeerType.REGULAR, }); testStore.addPeer(localPeer); @@ -107,6 +109,7 @@ const mockMediaStream = { kind: 'video', getSettings: jest.fn(() => ({ deviceId: 'video-device-id' })), addEventListener: jest.fn(() => {}), + removeEventListener: jest.fn(() => {}), }, ]), getAudioTracks: jest.fn(() => [ @@ -115,6 +118,7 @@ const mockMediaStream = { kind: 'audio', getSettings: jest.fn(() => ({ deviceId: 'audio-device-id' })), addEventListener: jest.fn(() => {}), + removeEventListener: jest.fn(() => {}), }, ]), addTrack: jest.fn(() => {}), @@ -204,7 +208,13 @@ const mockAudioContext = { return { stream: { getAudioTracks: jest.fn(() => [ - { id: 'audio-id', kind: 'audio', getSettings: jest.fn(() => ({ deviceId: 'audio-mock-device-id' })) }, + { + id: 'audio-id', + kind: 'audio', + getSettings: jest.fn(() => ({ deviceId: 'audio-mock-device-id' })), + addEventListener: jest.fn(() => {}), + removeEventListener: jest.fn(() => {}), + }, ]), }, }; @@ -231,8 +241,10 @@ describe('LocalTrackManager', () => { localPeer = new HMSLocalPeer({ name: 'test', role: hostRole, + type: HMSPeerType.REGULAR, }); testStore.addPeer(localPeer); + analyticsTimer = new AnalyticsTimer(); }); it('instantiates without any issues', () => { @@ -241,7 +253,7 @@ describe('LocalTrackManager', () => { testObserver, new DeviceManager(testStore, testEventBus), testEventBus, - new AnalyticsTimer(), + analyticsTimer, ); expect(manager).toBeDefined(); }); @@ -252,7 +264,7 @@ describe('LocalTrackManager', () => { testObserver, new DeviceManager(testStore, testEventBus), testEventBus, - new AnalyticsTimer(), + analyticsTimer, ); testStore.setKnownRoles(policyParams); await manager.getTracksToPublish({}); @@ -274,7 +286,7 @@ describe('LocalTrackManager', () => { testObserver, new DeviceManager(testStore, testEventBus), testEventBus, - new AnalyticsTimer(), + analyticsTimer, ); global.navigator.mediaDevices.getUserMedia = mockDenyGetUserMedia as any; testStore.setKnownRoles(policyParams); @@ -408,6 +420,7 @@ describe('LocalTrackManager', () => { localPeer = new HMSLocalPeer({ name: 'test', role: hostRole, + type: HMSPeerType.REGULAR, }); testStore.addPeer(localPeer); mockGetUserMedia.mockClear(); @@ -420,8 +433,10 @@ describe('LocalTrackManager', () => { id: 'video-track-id', kind: 'video', getSettings: () => ({ deviceId: 'video-device-id', groupId: 'video-group-id' }), - } as MediaStreamTrack, - 'regular', + addEventListener: jest.fn(() => {}), + removeEventListener: jest.fn(() => {}), + } as unknown as MediaStreamTrack, + HMSPeerType.REGULAR, testEventBus, ); localPeer.videoTrack = mockVideoTrack; @@ -433,7 +448,7 @@ describe('LocalTrackManager', () => { testObserver, new DeviceManager(testStore, testEventBus), testEventBus, - new AnalyticsTimer(), + analyticsTimer, ); testStore.setKnownRoles(policyParams); const tracksToPublish = await manager.getTracksToPublish({}); @@ -452,8 +467,10 @@ describe('LocalTrackManager', () => { id: 'video-track-id', kind: 'video', getSettings: () => ({ deviceId: 'video-device-id', groupId: 'video-group-id' }), - } as MediaStreamTrack, - 'regular', + addEventListener: jest.fn(() => {}), + removeEventListener: jest.fn(() => {}), + } as unknown as MediaStreamTrack, + HMSPeerType.REGULAR, testEventBus, ); @@ -462,7 +479,7 @@ describe('LocalTrackManager', () => { testObserver, new DeviceManager(testStore, testEventBus), testEventBus, - new AnalyticsTimer(), + analyticsTimer, ); testStore.setKnownRoles(policyParams); const tracksToPublish = await manager.getTracksToPublish({}); diff --git a/packages/hms-video-store/src/sdk/LocalTrackManager.ts b/packages/hms-video-store/src/sdk/LocalTrackManager.ts index 45e1f9a840..70103137db 100644 --- a/packages/hms-video-store/src/sdk/LocalTrackManager.ts +++ b/packages/hms-video-store/src/sdk/LocalTrackManager.ts @@ -7,9 +7,15 @@ import { ErrorCodes } from '../error/ErrorCodes'; import { ErrorFactory } from '../error/ErrorFactory'; import { HMSAction } from '../error/HMSAction'; import { HMSException } from '../error/HMSException'; -import { BuildGetMediaError, HMSGetMediaActions } from '../error/utils'; +import { BuildGetMediaError } from '../error/utils'; import { EventBus } from '../events/EventBus'; -import { HMSAudioCodec, HMSScreenShareConfig, HMSVideoCodec, ScreenCaptureHandleConfig } from '../interfaces'; +import { + HMSAudioCodec, + HMSAudioMode, + HMSScreenShareConfig, + HMSVideoCodec, + ScreenCaptureHandleConfig, +} from '../interfaces'; import InitialSettings from '../interfaces/settings'; import { HMSLocalAudioTrack, HMSLocalTrack, HMSLocalVideoTrack, HMSTrackType } from '../internal'; import { @@ -21,6 +27,7 @@ import { HMSVideoTrackSettingsBuilder, } from '../media/settings'; import { HMSLocalStream } from '../media/streams/HMSLocalStream'; +import { HMSTrackExceptionTrackType } from '../media/tracks/HMSTrackExceptionTrackType'; import ITransportObserver from '../transport/ITransportObserver'; import HMSLogger from '../utils/logger'; import { HMSAudioContextHandler } from '../utils/media'; @@ -31,6 +38,7 @@ const defaultSettings = { audioInputDeviceId: 'default', audioOutputDeviceId: 'default', videoDeviceId: 'default', + audioMode: HMSAudioMode.VOICE, }; type IFetchTrackOptions = boolean | 'empty'; interface IFetchAVTrackOptions { @@ -158,8 +166,39 @@ export class LocalTrackManager { nativeTracks.push(...this.getEmptyTracks(fetchTrackOptions)); return nativeTracks; } - - async getLocalScreen(partialConfig?: HMSScreenShareConfig) { + private async optimizeScreenShareConstraint(stream: MediaStream, constraints: MediaStreamConstraints) { + if (typeof constraints.video === 'boolean' || !constraints.video?.width || !constraints.video?.height) { + return; + } + const publishParams = this.store.getPublishParams(); + if (!publishParams || !publishParams.allowed?.includes('screen')) { + return; + } + const videoElement = document.createElement('video'); + videoElement.srcObject = stream; + videoElement.addEventListener('loadedmetadata', async () => { + const { videoWidth, videoHeight } = videoElement; + const screen = publishParams.screen; + const pixels = screen.width * screen.height; + const actualAspectRatio = videoWidth / videoHeight; + const currentAspectRatio = screen.width / screen.height; + if (actualAspectRatio > currentAspectRatio) { + const videoConstraint = constraints.video as MediaTrackConstraints; + const ratio = actualAspectRatio / currentAspectRatio; + const sqrt_ratio = Math.sqrt(ratio); + if (videoWidth * videoHeight > pixels) { + videoConstraint.width = videoWidth / sqrt_ratio; + videoConstraint.height = videoHeight / sqrt_ratio; + } else { + videoConstraint.height = videoHeight * sqrt_ratio; + videoConstraint.width = videoWidth * sqrt_ratio; + } + await stream.getVideoTracks()[0].applyConstraints(videoConstraint); + } + }); + } + // eslint-disable-next-line complexity + async getLocalScreen(partialConfig?: HMSScreenShareConfig, optimise = false) { const config = await this.getOrDefaultScreenshareConfig(partialConfig); const screenSettings = this.getScreenshareSettings(config.videoOnly); const constraints = { @@ -181,15 +220,24 @@ export class LocalTrackManager { googAutoGainControl: false, echoCancellation: false, }; + } else if (partialConfig?.forceCurrentTab && partialConfig.preferCurrentTab && partialConfig.cropElement) { + // only need if crop element with prefer and force current tab + constraints.audio = { + echoCancellation: true, + noiseSuppression: true, + }; } let stream; try { HMSLogger.d('retrieving screenshare with ', { config }, { constraints }); // @ts-ignore [https://github.com/microsoft/TypeScript/issues/33232] stream = (await navigator.mediaDevices.getDisplayMedia(constraints)) as MediaStream; + if (optimise) { + await this.optimizeScreenShareConstraint(stream, constraints); + } } catch (err) { HMSLogger.w(this.TAG, 'error in getting screenshare - ', err); - const error = BuildGetMediaError(err as Error, HMSGetMediaActions.SCREEN); + const error = BuildGetMediaError(err as Error, HMSTrackExceptionTrackType.SCREEN); this.eventBus.analytics.publish( AnalyticsEventFactory.publish({ error: error as Error, @@ -203,7 +251,14 @@ export class LocalTrackManager { const tracks: Array = []; const local = new HMSLocalStream(stream); const nativeVideoTrack = stream.getVideoTracks()[0]; - const videoTrack = new HMSLocalVideoTrack(local, nativeVideoTrack, 'screen', this.eventBus, screenSettings?.video); + const videoTrack = new HMSLocalVideoTrack( + local, + nativeVideoTrack, + 'screen', + this.eventBus, + screenSettings?.video, + this.store.getRoom(), + ); videoTrack.setSimulcastDefinitons(this.store.getSimulcastDefinitionsForPeer(this.store.getLocalPeer()!, 'screen')); try { @@ -224,6 +279,7 @@ export class LocalTrackManager { 'screen', this.eventBus, screenSettings?.audio, + this.store.getRoom(), ); tracks.push(audioTrack); } @@ -423,17 +479,17 @@ export class LocalTrackManager { } } - private getErrorType(videoError: boolean, audioError: boolean): HMSGetMediaActions { + getErrorType(videoError: boolean, audioError: boolean): HMSTrackExceptionTrackType { if (videoError && audioError) { - return HMSGetMediaActions.AV; + return HMSTrackExceptionTrackType.AUDIO_VIDEO; } if (videoError) { - return HMSGetMediaActions.VIDEO; + return HMSTrackExceptionTrackType.VIDEO; } if (audioError) { - return HMSGetMediaActions.AUDIO; + return HMSTrackExceptionTrackType.AUDIO; } - return HMSGetMediaActions.UNKNOWN; + return HMSTrackExceptionTrackType.AUDIO_VIDEO; } private getEmptyTracks(fetchTrackOptions: IFetchAVTrackOptions) { @@ -590,6 +646,7 @@ export class LocalTrackManager { 'regular', this.eventBus, settings.audio, + this.store.getRoom(), ); tracks.push(audioTrack); } @@ -601,6 +658,7 @@ export class LocalTrackManager { 'regular', this.eventBus, settings.video, + this.store.getRoom(), ); videoTrack.setSimulcastDefinitons( this.store.getSimulcastDefinitionsForPeer(this.store.getLocalPeer()!, 'regular'), diff --git a/packages/hms-video-store/src/sdk/RoleChangeManager.ts b/packages/hms-video-store/src/sdk/RoleChangeManager.ts index c4be09307b..c13b2519bf 100644 --- a/packages/hms-video-store/src/sdk/RoleChangeManager.ts +++ b/packages/hms-video-store/src/sdk/RoleChangeManager.ts @@ -1,6 +1,6 @@ import { Store } from './store'; import { DeviceManager } from '../device-manager'; -import { HMSRole } from '../interfaces'; +import { DeviceType, HMSRole } from '../interfaces'; import InitialSettings from '../interfaces/settings'; import { SimulcastLayers } from '../interfaces/simulcast-layers'; import { HMSPeerUpdate, HMSTrackUpdate, HMSUpdateListener } from '../interfaces/update-listener'; @@ -144,14 +144,49 @@ export default class RoleChangeManager { } private getSettings(): InitialSettings { - const initialSettings = this.store.getConfig()?.settings; + const { isAudioMuted, isVideoMuted } = this.getMutedStatus(); + const { audioInputDeviceId, audioOutputDeviceId } = this.getAudioDeviceSettings(); + const videoDeviceId = this.getVideoInputDeviceId(); + return { + isAudioMuted: isAudioMuted, + isVideoMuted: isVideoMuted, + audioInputDeviceId: audioInputDeviceId, + audioOutputDeviceId: audioOutputDeviceId, + videoDeviceId: videoDeviceId, + }; + } + private getMutedStatus(): { isAudioMuted: boolean; isVideoMuted: boolean } { + const initialSettings = this.store.getConfig()?.settings; return { isAudioMuted: initialSettings?.isAudioMuted ?? true, isVideoMuted: initialSettings?.isVideoMuted ?? true, - audioInputDeviceId: initialSettings?.audioInputDeviceId || 'default', - audioOutputDeviceId: initialSettings?.audioOutputDeviceId || 'default', - videoDeviceId: initialSettings?.videoDeviceId || 'default', }; } + + private getAudioDeviceSettings(): { audioInputDeviceId: string; audioOutputDeviceId: string } { + const initialSettings = this.store.getConfig()?.settings; + const audioInputDeviceId = + this.deviceManager.currentSelection[DeviceType.audioInput]?.deviceId || + initialSettings?.audioInputDeviceId || + 'default'; + const audioOutputDeviceId = + this.deviceManager.currentSelection[DeviceType.audioOutput]?.deviceId || + initialSettings?.audioOutputDeviceId || + 'default'; + + return { + audioInputDeviceId, + audioOutputDeviceId, + }; + } + + private getVideoInputDeviceId(): string { + const initialSettings = this.store.getConfig()?.settings; + return ( + this.deviceManager.currentSelection[DeviceType.videoInput]?.deviceId || + initialSettings?.videoDeviceId || + 'default' + ); + } } diff --git a/packages/hms-video-store/src/sdk/WakeLockManager.ts b/packages/hms-video-store/src/sdk/WakeLockManager.ts index 297f8c1af1..c29f78c497 100644 --- a/packages/hms-video-store/src/sdk/WakeLockManager.ts +++ b/packages/hms-video-store/src/sdk/WakeLockManager.ts @@ -19,6 +19,7 @@ export class WakeLockManager { HMSLogger.w(this.TAG, 'Error while releasing wake lock', `name=${error.name}, message=${error.message}`); } } + document?.removeEventListener('visibilitychange', this.visibilityHandler); this.wakeLock = null; }; diff --git a/packages/hms-video-store/src/sdk/index.ts b/packages/hms-video-store/src/sdk/index.ts index 526e349d18..ec8b0d1b0b 100644 --- a/packages/hms-video-store/src/sdk/index.ts +++ b/packages/hms-video-store/src/sdk/index.ts @@ -12,27 +12,33 @@ import { HMSAnalyticsLevel } from '../analytics/AnalyticsEventLevel'; import { AnalyticsEventsService } from '../analytics/AnalyticsEventsService'; import { AnalyticsTimer, TimedEvent } from '../analytics/AnalyticsTimer'; import { AudioSinkManager } from '../audio-sink-manager'; +import { PluginUsageTracker } from '../common/PluginUsageTracker'; import { DeviceManager } from '../device-manager'; import { AudioOutputManager } from '../device-manager/AudioOutputManager'; import { DeviceStorageManager } from '../device-manager/DeviceStorage'; +import { HMSDiagnosticsConnectivityListener } from '../diagnostics/interfaces'; +import { FeedbackService, HMSSessionFeedback, HMSSessionInfo } from '../end-call-feedback'; import { ErrorCodes } from '../error/ErrorCodes'; import { ErrorFactory } from '../error/ErrorFactory'; import { HMSAction } from '../error/HMSAction'; import { HMSException } from '../error/HMSException'; import { EventBus } from '../events/EventBus'; import { + HMSAudioCodec, HMSChangeMultiTrackStateParams, HMSConfig, HMSConnectionQualityListener, HMSDeviceChangeEvent, HMSFrameworkInfo, HMSMessageInput, + HMSPeerType, HMSPlaylistSettings, HMSPlaylistType, HMSPreviewConfig, HMSRole, HMSRoleChangeRequest, HMSScreenShareConfig, + HMSVideoCodec, TokenRequest, TokenRequestOptions, } from '../interfaces'; @@ -46,7 +52,8 @@ import { HMSPreviewListener } from '../interfaces/preview-listener'; import { RTMPRecordingConfig } from '../interfaces/rtmp-recording-config'; import InitialSettings from '../interfaces/settings'; import { HMSAudioListener, HMSPeerUpdate, HMSTrackUpdate, HMSUpdateListener } from '../interfaces/update-listener'; -import { PlaylistManager } from '../internal'; +import { PlaylistManager, TranscriptionConfig } from '../internal'; +import { HMSAudioTrackSettingsBuilder, HMSVideoTrackSettingsBuilder } from '../media/settings'; import { HMSLocalStream } from '../media/streams/HMSLocalStream'; import { HMSLocalAudioTrack, @@ -57,21 +64,30 @@ import { HMSTrackType, HMSVideoTrack, } from '../media/tracks'; -import { HMSNotificationMethod, PeerLeaveRequestNotification, SendMessage } from '../notification-manager'; +import { + HMSNotificationMethod, + PeerLeaveRequestNotification, + PeerNotificationInfo, + SendMessage, +} from '../notification-manager'; import { createRemotePeer } from '../notification-manager/managers/utils'; import { NotificationManager } from '../notification-manager/NotificationManager'; +import { DebugInfo } from '../schema'; import { SessionStore } from '../session-store'; import { InteractivityCenter } from '../session-store/interactivity-center'; -import { InitConfig } from '../signal/init/models'; +import { InitConfig, InitFlags } from '../signal/init/models'; import { + FindPeerByNameRequestParams, HLSRequestParams, HLSTimedMetadataParams, HLSVariant, StartRTMPOrRecordingRequestParams, + StartTranscriptionRequestParams, } from '../signal/interfaces'; import HMSTransport from '../transport'; import ITransportObserver from '../transport/ITransportObserver'; import { TransportState } from '../transport/models/TransportState'; +import { getAnalyticsDeviceId } from '../utils/analytics-deviceId'; import { DEFAULT_PLAYLIST_AUDIO_BITRATE, DEFAULT_PLAYLIST_VIDEO_BITRATE, @@ -83,7 +99,7 @@ import HMSLogger, { HMSLogLevel } from '../utils/logger'; import { HMSAudioContextHandler } from '../utils/media'; import { isNode } from '../utils/support'; import { workerSleep } from '../utils/timer-utils'; -import { validateMediaDevicesExistence, validateRTCPeerConnection } from '../utils/validations'; +import { validateMediaDevicesExistence, validatePublishParams, validateRTCPeerConnection } from '../utils/validations'; const INITIAL_STATE = { published: false, @@ -98,19 +114,21 @@ const INITIAL_STATE = { export class HMSSdk implements HMSInterface { private transport!: HMSTransport; private readonly TAG = '[HMSSdk]:'; - private listener?: HMSUpdateListener; + public listener?: HMSUpdateListener; private errorListener?: IErrorListener; private deviceChangeListener?: DeviceChangeListener; private audioListener?: HMSAudioListener; public store!: Store; private notificationManager?: NotificationManager; - private deviceManager!: DeviceManager; + /** @internal */ + public deviceManager!: DeviceManager; private audioSinkManager!: AudioSinkManager; private playlistManager!: PlaylistManager; private audioOutput!: AudioOutputManager; private transportState: TransportState = TransportState.Disconnected; private roleChangeManager?: RoleChangeManager; - private localTrackManager!: LocalTrackManager; + /** @internal */ + public localTrackManager!: LocalTrackManager; private analyticsEventsService!: AnalyticsEventsService; private analyticsTimer = new AnalyticsTimer(); private eventBus!: EventBus; @@ -118,8 +136,18 @@ export class HMSSdk implements HMSInterface { private wakeLockManager!: WakeLockManager; private sessionStore!: SessionStore; private interactivityCenter!: InteractivityCenter; + private pluginUsageTracker!: PluginUsageTracker; private sdkState = { ...INITIAL_STATE }; private frameworkInfo?: HMSFrameworkInfo; + private isDiagnostics = false; + /** + * will be set post join + * this will not be reset on leave but after feedback success + * we will just clean token after successful submit feedback + * will be replaced when a newer join happens. + */ + private sessionPeerInfo?: HMSSessionInfo; + private playlistSettings: HMSPlaylistSettings = { video: { bitrate: DEFAULT_PLAYLIST_VIDEO_BITRATE, @@ -129,6 +157,33 @@ export class HMSSdk implements HMSInterface { }, }; + private setSessionPeerInfo(websocketURL: string, peer?: HMSLocalPeer) { + const room = this.store.getRoom(); + if (!peer || !room) { + HMSLogger.e(this.TAG, 'setSessionPeerInfo> Local peer or room is undefined'); + return; + } + this.sessionPeerInfo = { + peer: { + peer_id: peer.peerId, + role: peer.role?.name, + joined_at: peer.joinedAt?.valueOf() || 0, + room_name: room.name, + session_started_at: room.startedAt?.valueOf() || 0, + user_data: peer.customerUserId, + user_name: peer.name, + template_id: room.templateId, + session_id: room.sessionId, + token: this.store.getConfig()?.authToken, + }, + agent: this.store.getUserAgent(), + device_id: getAnalyticsDeviceId(), + cluster: { + websocket_url: websocketURL, + }, + timestamp: Date.now(), + }; + } private initNotificationManager() { if (!this.notificationManager) { this.notificationManager = new NotificationManager( @@ -141,7 +196,13 @@ export class HMSSdk implements HMSInterface { } } - private initStoreAndManagers() { + /** @internal */ + initStoreAndManagers(listener: HMSPreviewListener | HMSUpdateListener | HMSDiagnosticsConnectivityListener) { + this.listener = listener as unknown as HMSUpdateListener; + this.errorListener = listener; + this.deviceChangeListener = listener; + this.store?.setErrorListener(this.errorListener); + if (this.sdkState.isInitialised) { /** * Set listener after both join and preview, since they can have different listeners @@ -149,12 +210,15 @@ export class HMSSdk implements HMSInterface { this.notificationManager?.setListener(this.listener); this.audioSinkManager.setListener(this.listener); this.interactivityCenter.setListener(this.listener); + this.transport.setListener(this.listener); return; } this.sdkState.isInitialised = true; this.store = new Store(); + this.store.setErrorListener(this.errorListener); this.eventBus = new EventBus(); + this.pluginUsageTracker = new PluginUsageTracker(this.eventBus); this.wakeLockManager = new WakeLockManager(); this.networkTestManager = new NetworkTestManager(this.eventBus, this.listener); this.playlistManager = new PlaylistManager(this, this.eventBus); @@ -178,16 +242,22 @@ export class HMSSdk implements HMSInterface { this.eventBus, this.analyticsEventsService, this.analyticsTimer, + this.pluginUsageTracker, ); + // add diagnostics callbacks if present + if ('onInitSuccess' in listener) { + this.transport.setConnectivityListener(listener); + } this.sessionStore = new SessionStore(this.transport); this.interactivityCenter = new InteractivityCenter(this.transport, this.store, this.listener); - /** * Note: Subscribe to events here right after creating stores and managers * to not miss events that are published before the handlers are subscribed. */ this.eventBus.analytics.subscribe(this.sendAnalyticsEvent); this.eventBus.deviceChange.subscribe(this.handleDeviceChange); + this.eventBus.localVideoUnmutedNatively.subscribe(this.unpauseRemoteVideoTracks); + this.eventBus.localAudioUnmutedNatively.subscribe(this.unpauseRemoteVideoTracks); this.eventBus.audioPluginFailed.subscribe(this.handleAudioPluginError); } @@ -211,6 +281,21 @@ export class HMSSdk implements HMSInterface { return this.transport?.getWebrtcInternals(); } + getDebugInfo(): DebugInfo | undefined { + if (!this.transport) { + HMSLogger.e(this.TAG, `Transport is not defined`); + throw new Error('getDebugInfo can only be called after join'); + } + const websocketURL = this.transport.getWebsocketEndpoint(); + const enabledFlags = Object.values(InitFlags).filter(flag => this.transport.isFlagEnabled(flag)); + const initEndpoint = this.store.getConfig()?.initEndpoint; + return { + websocketURL, + enabledFlags, + initEndpoint, + }; + } + getSessionStore() { return this.sessionStore; } @@ -231,6 +316,10 @@ export class HMSSdk implements HMSInterface { return this.store.getRoom()?.hls; } + getTranscriptionState() { + return this.store.getRoom()?.transcriptions; + } + getTemplateAppData() { return this.store.getTemplateAppData(); } @@ -363,13 +452,6 @@ export class HMSSdk implements HMSInterface { this.analyticsTimer.start(TimedEvent.PREVIEW); await this.setUpPreview(config, listener); - // Request permissions and populate devices before waiting for policy - if (config.alwaysRequestPermissions) { - this.localTrackManager.requestPermissions().then(async () => { - await this.initDeviceManagers(); - }); - } - let initSuccessful = false; let networkTestFinished = false; const timerId = setTimeout(() => { @@ -385,7 +467,22 @@ export class HMSSdk implements HMSInterface { this.localPeer.asRole = newRole || this.localPeer.role; } const tracks = await this.localTrackManager.getTracksToPublish(config.settings); - tracks.forEach(track => this.setLocalPeerTrack(track)); + tracks.forEach(track => { + this.setLocalPeerTrack(track); + if (track.isTrackNotPublishing()) { + const error = ErrorFactory.TracksErrors.NoDataInTrack( + `${track.type} track has no data. muted: ${track.nativeTrack.muted}, readyState: ${track.nativeTrack.readyState}`, + ); + HMSLogger.e(this.TAG, error); + this.sendAnalyticsEvent( + AnalyticsEventFactory.publish({ + devices: this.deviceManager.getDevices(), + error: error, + }), + ); + this.listener?.onError(error); + } + }); this.localPeer?.audioTrack && this.initPreviewTrackAudioLevelMonitor(); await this.initDeviceManagers(); this.sdkState.isPreviewInProgress = false; @@ -398,16 +495,9 @@ export class HMSSdk implements HMSInterface { resolve(); }; - const errorHandler = (ex?: HMSException) => { - this.analyticsTimer.end(TimedEvent.PREVIEW); - ex && this.errorListener?.onError(ex); - this.sendPreviewAnalyticsEvent(ex); - this.sdkState.isPreviewInProgress = false; - reject(ex as HMSException); - }; - this.eventBus.policyChange.subscribeOnce(policyHandler); - this.eventBus.leave.subscribeOnce(errorHandler); + this.eventBus.leave.subscribeOnce(this.handlePreviewError); + this.eventBus.leave.subscribeOnce(ex => reject(ex as HMSException)); this.transport .preview( @@ -416,6 +506,7 @@ export class HMSSdk implements HMSInterface { this.localPeer!.peerId, { name: config.userName, metaData: config.metaData || '' }, config.autoVideoSubscribe, + config.iceServers, ) .then((initConfig: InitConfig | void) => { initSuccessful = true; @@ -426,10 +517,20 @@ export class HMSSdk implements HMSInterface { }); } }) - .catch(errorHandler); + .catch(ex => { + this.handlePreviewError(ex); + reject(ex); + }); }); } + private handlePreviewError = (ex?: HMSException) => { + this.analyticsTimer.end(TimedEvent.PREVIEW); + ex && this.errorListener?.onError(ex); + this.sendPreviewAnalyticsEvent(ex); + this.sdkState.isPreviewInProgress = false; + }; + private async midCallPreview(asRole?: string, settings?: InitialSettings): Promise { if (!this.localPeer || this.transportState !== TransportState.Joined) { throw ErrorFactory.GenericErrors.NotConnected(HMSAction.VALIDATION, 'Not connected - midCallPreview'); @@ -498,6 +599,7 @@ export class HMSSdk implements HMSInterface { this.errorListener?.onError(error); }; + // eslint-disable-next-line complexity async join(config: HMSConfig, listener: HMSUpdateListener) { validateMediaDevicesExistence(); validateRTCPeerConnection(); @@ -506,13 +608,14 @@ export class HMSSdk implements HMSInterface { throw ErrorFactory.GenericErrors.NotReady(HMSAction.JOIN, "Preview is in progress, can't join"); } + // remove terminal error handling from preview(do not send preview.failed after join on disconnect) + this.eventBus?.leave?.unsubscribe(this.handlePreviewError); this.analyticsTimer.start(TimedEvent.JOIN); this.sdkState.isJoinInProgress = true; const { roomId, userId, role } = decodeJWT(config.authToken); const previewRole = this.localPeer?.asRole?.name || this.localPeer?.role?.name; this.networkTestManager?.stop(); - this.listener = listener; this.commonSetup(config, roomId, listener); this.removeDevicesFromConfig(config); this.store.setConfig(config); @@ -556,6 +659,7 @@ export class HMSSdk implements HMSInterface { { name: config.userName, metaData: config.metaData! }, config.initEndpoint!, config.autoVideoSubscribe, + config.iceServers, ); HMSLogger.d(this.TAG, `✅ Joined room ${roomId}`); this.analyticsTimer.start(TimedEvent.PEER_LIST); @@ -584,6 +688,8 @@ export class HMSSdk implements HMSInterface { private cleanup() { this.cleanDeviceManagers(); this.eventBus.analytics.unsubscribe(this.sendAnalyticsEvent); + this.eventBus.localVideoUnmutedNatively.unsubscribe(this.unpauseRemoteVideoTracks); + this.eventBus.localAudioUnmutedNatively.unsubscribe(this.unpauseRemoteVideoTracks); this.analyticsTimer.cleanup(); DeviceStorageManager.cleanup(); this.playlistManager.cleanup(); @@ -623,16 +729,19 @@ export class HMSSdk implements HMSInterface { await workerSleep(100); } const roomId = room.id; + // setSessionJoin + this.setSessionPeerInfo(this.transport.getWebsocketEndpoint() || '', this.localPeer); this.networkTestManager?.stop(); this.eventBus.leave.publish(error); - HMSLogger.d(this.TAG, `⏳ Leaving room ${roomId}`); + const peerId = this.localPeer?.peerId; + HMSLogger.d(this.TAG, `⏳ Leaving room ${roomId}, peerId=${peerId}`); // browsers often put limitation on amount of time a function set on window onBeforeUnload can take in case of // tab refresh or close. Therefore prioritise the leave action over anything else, if tab is closed/refreshed // we would want leave to succeed to stop stucked peer for others. The followup cleanup however is important // for cases where uses stays on the page post leave. await this.transport?.leave(notifyServer); this.cleanup(); - HMSLogger.d(this.TAG, `✅ Left room ${roomId}`); + HMSLogger.d(this.TAG, `✅ Left room ${roomId}, peerId=${peerId}`); } } @@ -706,11 +815,11 @@ export class HMSSdk implements HMSInterface { let recipientPeer = this.store.getPeerById(peerId); if (!recipientPeer) { if (isLargeRoom) { - const { peers } = await this.transport.signal.findPeers({ peers: [peerId], limit: 1 }); - if (peers.length === 0) { + const peer = await this.transport.signal.getPeer({ peer_id: peerId }); + if (!peer) { throw ErrorFactory.GenericErrors.ValidationFailed('Invalid peer - peer not present in the room', peerId); } - recipientPeer = createRemotePeer(peers[0], this.store); + recipientPeer = createRemotePeer(peer, this.store); } else { throw ErrorFactory.GenericErrors.ValidationFailed('Invalid peer - peer not present in the room', peerId); } @@ -718,6 +827,68 @@ export class HMSSdk implements HMSInterface { return await this.sendMessageInternal({ message, recipientPeer, type }); } + async submitSessionFeedback(feedback: HMSSessionFeedback, eventEndpoint?: string) { + if (!this.sessionPeerInfo) { + HMSLogger.e(this.TAG, 'submitSessionFeedback> session is undefined'); + throw new Error('session is undefined'); + } + const token = this.sessionPeerInfo.peer.token; + if (!token) { + HMSLogger.e(this.TAG, 'submitSessionFeedback> token is undefined'); + throw new Error('Internal error, token is not present'); + } + try { + await FeedbackService.sendFeedback({ + token: token, + info: this.sessionPeerInfo, + feedback, + eventEndpoint, + }); + HMSLogger.i(this.TAG, 'submitSessionFeedback> submitted feedback'); + this.sessionPeerInfo = undefined; + } catch (e) { + HMSLogger.e(this.TAG, 'submitSessionFeedback> error occured ', e); + throw new Error('Unable to submit feedback'); + } + } + async getPeer(peerId: string) { + const response = await this.transport.signal.getPeer({ peer_id: peerId }); + if (response) { + return createRemotePeer(response, this.store); + } + return undefined; + } + + async findPeerByName({ query, limit = 10, offset }: FindPeerByNameRequestParams) { + const { + peers, + offset: responseOffset, + eof, + } = await this.transport.signal.findPeerByName({ query: query?.toLowerCase(), limit, offset }); + if (peers.length > 0) { + return { + offset: responseOffset, + eof, + peers: peers.map(peerInfo => { + return createRemotePeer( + { + peer_id: peerInfo.peer_id, + role: peerInfo.role, + groups: [], + info: { + name: peerInfo.name, + data: '', + user_id: '', + type: peerInfo.type, + }, + } as PeerNotificationInfo, + this.store, + ); + }), + }; + } + return { offset: responseOffset, peers: [] }; + } private async sendMessageInternal({ recipientRoles, recipientPeer, type = 'chat', message }: HMSMessageInput) { if (message.replace(/\u200b/g, ' ').trim() === '') { @@ -766,6 +937,9 @@ export class HMSSdk implements HMSInterface { }); return; } + this.transport.setOnScreenshareStop(() => { + this.stopEndedScreenshare(onStop); + }); await this.transport.publish(tracks); tracks.forEach(track => { track.peerId = this.localPeer?.peerId; @@ -809,7 +983,8 @@ export class HMSSdk implements HMSInterface { const TrackKlass = type === 'audio' ? HMSLocalAudioTrack : HMSLocalVideoTrack; const hmsTrack = new TrackKlass(stream, track, source, this.eventBus); - this.setPlaylistSettings({ + await this.applySettings(hmsTrack); + await this.setPlaylistSettings({ track, hmsTrack, source, @@ -861,6 +1036,11 @@ export class HMSSdk implements HMSInterface { this.notificationManager?.setConnectionQualityListener(qualityListener); } + /** @internal */ + setIsDiagnostics(isDiagnostics: boolean) { + this.isDiagnostics = isDiagnostics; + } + async changeRole(forPeerId: string, toRole: string, force = false) { await this.transport?.signal.requestRoleChange({ requested_for: forPeerId, @@ -993,6 +1173,35 @@ export class HMSSdk implements HMSInterface { await this.transport?.signal.stopHLSStreaming(); } + async startTranscription(params: TranscriptionConfig) { + if (!this.localPeer) { + throw ErrorFactory.GenericErrors.NotConnected( + HMSAction.VALIDATION, + 'No local peer present, cannot start transcriptions', + ); + } + const transcriptionParams: StartTranscriptionRequestParams = { + mode: params.mode, + }; + await this.transport?.signal.startTranscription(transcriptionParams); + } + + async stopTranscription(params: TranscriptionConfig) { + if (!this.localPeer) { + throw ErrorFactory.GenericErrors.NotConnected( + HMSAction.VALIDATION, + 'No local peer present, cannot stop transcriptions', + ); + } + if (!params) { + throw ErrorFactory.GenericErrors.Signalling(HMSAction.VALIDATION, 'No mode is passed to stop the transcription'); + } + const transcriptionParams: StartTranscriptionRequestParams = { + mode: params.mode, + }; + await this.transport?.signal.stopTranscription(transcriptionParams); + } + async sendHLSTimedMetadata(metadataList: HLSTimedMetadata[]) { this.validateJoined('sendHLSTimedMetadata'); if (metadataList.length > 0) { @@ -1141,13 +1350,28 @@ export class HMSSdk implements HMSInterface { } private handleLocalRoleUpdate = async ({ oldRole, newRole }: { oldRole: HMSRole; newRole: HMSRole }) => { + this.deviceManager.currentSelection = this.deviceManager.getCurrentSelection(); await this.transport.handleLocalRoleUpdate({ oldRole, newRole }); await this.roleChangeManager?.handleLocalPeerRoleUpdate({ oldRole, newRole }); + await this.interactivityCenter.whiteboard.handleLocalRoleUpdate(); }; private async setAndPublishTracks(tracks: HMSLocalTrack[]) { for (const track of tracks) { await this.transport.publish([track]); + if (track.isTrackNotPublishing()) { + const error = ErrorFactory.TracksErrors.NoDataInTrack( + `${track.type} track has no data. muted: ${track.nativeTrack.muted}, readyState: ${track.nativeTrack.readyState}`, + ); + HMSLogger.e(this.TAG, error); + this.sendAnalyticsEvent( + AnalyticsEventFactory.publish({ + devices: this.deviceManager.getDevices(), + error: error, + }), + ); + this.listener?.onError(error); + } this.setLocalPeerTrack(track); this.listener?.onTrackUpdate(HMSTrackUpdate.TRACK_ADDED, track, this.localPeer!); } @@ -1188,7 +1412,8 @@ export class HMSSdk implements HMSInterface { this.audioSinkManager.cleanup(); } - private initPreviewTrackAudioLevelMonitor() { + /** @internal */ + initPreviewTrackAudioLevelMonitor() { const localAudioTrack = this.localPeer?.audioTrack; localAudioTrack?.initAudioLevelMonitor(); this.eventBus.trackAudioLevelUpdate.subscribe(audioLevelUpdate => { @@ -1238,6 +1463,7 @@ export class HMSSdk implements HMSInterface { * Init store and other managers, setup listeners, create local peer, room * @param {HMSConfig} config * @param {HMSPreviewListener} listener + * @returns {Promise} - resolves when store is initialised */ private async setUpPreview(config: HMSPreviewConfig, listener: HMSPreviewListener) { this.listener = listener as unknown as HMSUpdateListener; @@ -1296,6 +1522,7 @@ export class HMSSdk implements HMSInterface { role: policy, // default value is the original role if user didn't pass asRole in config asRole: asRolePolicy || policy, + type: HMSPeerType.REGULAR, }); this.store.addPeer(localPeer); @@ -1312,10 +1539,8 @@ export class HMSSdk implements HMSInterface { if (!config.initEndpoint) { config.initEndpoint = 'https://prod-init.100ms.live'; } - this.errorListener = listener; - this.deviceChangeListener = listener; - this.initStoreAndManagers(); - this.store.setErrorListener(this.errorListener); + + this.initStoreAndManagers(listener); if (!this.store.getRoom()) { this.store.setRoom(new HMSRoom(roomId)); } @@ -1342,7 +1567,8 @@ export class HMSSdk implements HMSInterface { * @returns */ private async getScreenshareTracks(onStop: () => void, config?: HMSScreenShareConfig) { - const [videoTrack, audioTrack] = await this.localTrackManager.getLocalScreen(config); + const isOptimizedScreenShare = this.transport.isFlagEnabled(InitFlags.FLAG_SCALE_SCREENSHARE_BASED_ON_PIXELS); + const [videoTrack, audioTrack] = await this.localTrackManager.getLocalScreen(config, isOptimizedScreenShare); const handleEnded = () => { this.stopEndedScreenshare(onStop); @@ -1371,13 +1597,19 @@ export class HMSSdk implements HMSInterface { return tracks; } + private unpauseRemoteVideoTracks = () => { + this.store.getRemoteVideoTracks().forEach(track => track.handleTrackUnmute()); + }; + private sendAudioPresenceFailed = () => { const error = ErrorFactory.TracksErrors.NoAudioDetected(HMSAction.PREVIEW); HMSLogger.w(this.TAG, 'Audio Presence Failure', this.transportState, error); // this.sendAnalyticsEvent( // AnalyticsEventFactory.audioDetectionFail(error, this.deviceManager.getCurrentSelection().audioInput), // ); - // this.listener?.onError(error); + if (this.isDiagnostics) { + this.listener?.onError(error); + } }; private sendJoinAnalyticsEvent = (is_preview_called = false, error?: HMSException) => { @@ -1403,6 +1635,10 @@ export class HMSSdk implements HMSInterface { }; private sendAnalyticsEvent = (event: AnalyticsEvent) => { + // don't send analytics for diagnostics + if (this.isDiagnostics) { + return; + } this.analyticsEventsService.queue(event).flush(); }; @@ -1413,4 +1649,42 @@ export class HMSSdk implements HMSInterface { this.playlistManager.stop(HMSPlaylistType.video); } } + + // eslint-disable-next-line complexity + private async applySettings(track: HMSLocalTrack) { + validatePublishParams(this.store); + const publishParams = this.store.getPublishParams(); + // this is not needed but added for avoiding ? later + if (!publishParams) { + return; + } + if (track instanceof HMSLocalVideoTrack) { + const publishKey = track.source === 'regular' ? 'video' : track.source === 'screen' ? 'screen' : ''; + if (!publishKey || !publishParams.allowed.includes(publishKey)) { + return; + } + const video = publishParams[publishKey]; + if (!video) { + return; + } + const settings = new HMSVideoTrackSettingsBuilder() + .codec(video.codec as HMSVideoCodec) + .maxBitrate(video.bitRate) + .maxFramerate(video.frameRate) + .setWidth(video.width) + .setHeight(video.height) + .build(); + + await track.setSettings(settings); + } else if (track instanceof HMSLocalAudioTrack) { + if (!publishParams.allowed.includes('audio')) { + return; + } + const settings = new HMSAudioTrackSettingsBuilder() + .codec(publishParams.audio.codec as HMSAudioCodec) + .maxBitrate(publishParams.audio.bitRate) + .build(); + await track.setSettings(settings); + } + } } diff --git a/packages/hms-video-store/src/sdk/models/HMSRoom.ts b/packages/hms-video-store/src/sdk/models/HMSRoom.ts index 23368d09d3..c23b03e0a9 100644 --- a/packages/hms-video-store/src/sdk/models/HMSRoom.ts +++ b/packages/hms-video-store/src/sdk/models/HMSRoom.ts @@ -1,4 +1,4 @@ -import { HMSHLS, HMSRecording, HMSRoom, HMSRTMP } from '../../interfaces/room'; +import { HMSHLS, HMSRecording, HMSRoom, HMSRTMP, HMSTranscriptionInfo } from '../../interfaces/room'; export default class Room implements HMSRoom { id: string; @@ -14,14 +14,13 @@ export default class Room implements HMSRoom { description?: string; max_size?: number; large_room_optimization?: boolean; - /** - * @alpha - */ + transcriptions?: HMSTranscriptionInfo[] = []; isEffectsEnabled?: boolean; - /** - * @alpha - */ + disableNoneLayerRequest?: boolean; + isVBEnabled?: boolean; effectsKey?: string; + isHipaaEnabled?: boolean; + isNoiseCancellationEnabled?: boolean; constructor(id: string) { this.id = id; diff --git a/packages/hms-video-store/src/sdk/models/peer/HMSPeer.ts b/packages/hms-video-store/src/sdk/models/peer/HMSPeer.ts index b119db956f..54bf7547c8 100644 --- a/packages/hms-video-store/src/sdk/models/peer/HMSPeer.ts +++ b/packages/hms-video-store/src/sdk/models/peer/HMSPeer.ts @@ -1,4 +1,5 @@ import { HMSPeer as IHMSPeer } from '../../../interfaces/peer'; +import { HMSPeerType } from '../../../interfaces/peer/hms-peer'; import { HMSRole } from '../../../interfaces/role'; import { HMSAudioTrack, HMSTrack, HMSVideoTrack } from '../../../media/tracks'; import { HAND_RAISE_GROUP_NAME } from '../../../utils/constants'; @@ -16,6 +17,7 @@ export type HMSPeerInit = { groups?: string[]; realtime?: boolean; isHandRaised?: boolean; + type: HMSPeerType; }; export class HMSPeer implements IHMSPeer { @@ -32,8 +34,20 @@ export class HMSPeer implements IHMSPeer { networkQuality?: number; groups?: string[]; realtime?: boolean; + type: HMSPeerType; - constructor({ peerId, name, isLocal, customerUserId, metadata, role, joinedAt, groups, realtime }: HMSPeerInit) { + constructor({ + peerId, + name, + isLocal, + customerUserId, + metadata, + role, + joinedAt, + groups, + realtime, + type, + }: HMSPeerInit) { this.name = name; this.peerId = peerId; this.isLocal = isLocal; @@ -42,6 +56,7 @@ export class HMSPeer implements IHMSPeer { this.joinedAt = joinedAt; this.groups = groups; this.realtime = realtime; + this.type = type; if (role) { this.role = role; @@ -68,6 +83,7 @@ export class HMSPeer implements IHMSPeer { updateNetworkQuality(quality: number) { this.networkQuality = quality; } + /** * @internal */ diff --git a/packages/hms-video-store/src/sdk/models/peer/peer.test.ts b/packages/hms-video-store/src/sdk/models/peer/peer.test.ts index c459790c5a..f9c51519e0 100644 --- a/packages/hms-video-store/src/sdk/models/peer/peer.test.ts +++ b/packages/hms-video-store/src/sdk/models/peer/peer.test.ts @@ -1,5 +1,6 @@ import { HMSLocalPeer } from './HMSLocalPeer'; import { HMSRemotePeer } from './HMSRemotePeer'; +import { HMSPeerType } from '../../../internal'; import { PeerNotification } from '../../../notification-manager'; import decodeJWT from '../../../utils/jwt'; @@ -45,6 +46,7 @@ describe('HMSLocalPeer', () => { name: 'John Doe', role: getParamsForRole(role), customerUserId: userId, + type: HMSPeerType.REGULAR as HMSPeerType, }; const peer = new HMSLocalPeer(params); @@ -80,6 +82,7 @@ describe('HMSRemotPeer', () => { name: 'John Doe', data: 'data', user_id: 'customer_user_id', + type: HMSPeerType.REGULAR, }, role: 'viewer', tracks: {}, @@ -91,6 +94,7 @@ describe('HMSRemotPeer', () => { role: getParamsForRole(peerInfo.role), customerUserId: peerInfo.info.user_id, metadata: peerInfo.info.data, + type: HMSPeerType.REGULAR, }); it('should be constructed using params', () => { diff --git a/packages/hms-video-store/src/sdk/store/Store.ts b/packages/hms-video-store/src/sdk/store/Store.ts index aea6fb4393..0ab18458fd 100644 --- a/packages/hms-video-store/src/sdk/store/Store.ts +++ b/packages/hms-video-store/src/sdk/store/Store.ts @@ -3,7 +3,15 @@ import { HTTPAnalyticsTransport } from '../../analytics/HTTPAnalyticsTransport'; import { DeviceStorageManager } from '../../device-manager/DeviceStorage'; import { ErrorFactory } from '../../error/ErrorFactory'; import { HMSAction } from '../../error/HMSAction'; -import { HMSConfig, HMSFrameworkInfo, HMSPermissionType, HMSPoll, HMSSpeaker, HMSWhiteboard } from '../../interfaces'; +import { + HMSConfig, + HMSFrameworkInfo, + HMSPermissionType, + HMSPoll, + HMSSpeaker, + HMSTranscriptionMode, + HMSWhiteboard, +} from '../../interfaces'; import { SelectedDevices } from '../../interfaces/devices'; import { IErrorListener } from '../../interfaces/error-listener'; import { @@ -23,13 +31,21 @@ import { HMSTrackType, HMSVideoTrack, } from '../../media/tracks'; -import { PolicyParams } from '../../notification-manager'; +import { + NoiseCancellationPlugin, + Plugins, + PolicyParams, + TranscriptionPluginPermissions, + WhiteBoardPluginPermissions, +} from '../../notification-manager'; +import HMSLogger from '../../utils/logger'; import { ENV } from '../../utils/support'; import { createUserAgent } from '../../utils/user-agent'; import HMSRoom from '../models/HMSRoom'; import { HMSLocalPeer, HMSPeer, HMSRemotePeer } from '../models/peer'; class Store { + private TAG = '[Store]:'; private room?: HMSRoom; private knownRoles: KnownRoles = {}; private localPeerId?: string; @@ -59,6 +75,15 @@ class Store { this.simulcastEnabled = enabled; } + removeRemoteTracks() { + this.tracks.forEach(track => { + if (track instanceof HMSRemoteAudioTrack || track instanceof HMSRemoteVideoTrack) { + this.removeTrack(track); + delete this.peerTrackStates[track.trackId]; + } + }); + } + getEnv() { return this.env; } @@ -264,6 +289,10 @@ class Store { this.peerTrackStates[trackStateEntry.trackInfo.track_id] = trackStateEntry; } + removeTrackState(trackId: string) { + delete this.peerTrackStates[trackId]; + } + removePeer(peerId: string) { if (this.localPeerId === peerId) { this.localPeerId = undefined; @@ -373,6 +402,10 @@ class Store { this.whiteboards.set(whiteboard.id, whiteboard); } + getWhiteboards() { + return this.whiteboards; + } + getWhiteboard(id?: string): HMSWhiteboard | undefined { return id ? this.whiteboards.get(id) : this.whiteboards.values().next().value; } @@ -410,30 +443,72 @@ class Store { return; } - const addPermissionToRole = ( - role: string, - pluginName: keyof PolicyParams['plugins'], - permission: HMSPermissionType, - ) => { - const rolePermissions = this.knownRoles[role].permissions; - if (!rolePermissions[pluginName]) { - rolePermissions[pluginName] = []; - } - rolePermissions[pluginName]?.push(permission); - }; - Object.keys(plugins).forEach(plugin => { const pluginName = plugin as keyof PolicyParams['plugins']; - if (!plugins[pluginName]) { - return; + switch (pluginName) { + case Plugins.WHITEBOARD: { + this.addWhiteboardPluginToRole(plugins[pluginName]); + break; + } + case Plugins.TRANSCRIPTIONS: { + this.addTranscriptionsPluginToRole(plugins[pluginName]); + break; + } + case Plugins.NOISE_CANCELLATION: { + this.handleNoiseCancellationPlugin(plugins[pluginName]); + break; + } + default: { + break; + } } - - const permissions = plugins[pluginName].permissions; - permissions?.admin?.forEach(role => addPermissionToRole(role, pluginName, 'admin')); - permissions?.reader?.forEach(role => addPermissionToRole(role, pluginName, 'read')); - permissions?.writer?.forEach(role => addPermissionToRole(role, pluginName, 'write')); }); } + private addPermissionToRole = ( + role: string, + pluginName: keyof PolicyParams['plugins'], + permission: HMSPermissionType, + mode?: HMSTranscriptionMode, + ) => { + if (!this.knownRoles[role]) { + HMSLogger.d(this.TAG, `role ${role} is not present in given roles`, this.knownRoles); + return; + } + const rolePermissions = this.knownRoles[role].permissions; + if (pluginName === Plugins.TRANSCRIPTIONS && mode) { + // currently only admin is allowed, so no issue + rolePermissions[pluginName] = { + ...rolePermissions[pluginName], + [mode]: [permission], + }; + } else if (pluginName === Plugins.WHITEBOARD) { + if (!rolePermissions[pluginName]) { + rolePermissions[pluginName] = []; + } + rolePermissions[pluginName]?.push(permission); + } + }; + private addWhiteboardPluginToRole = (plugin?: WhiteBoardPluginPermissions) => { + const permissions = plugin?.permissions; + permissions?.admin?.forEach(role => this.addPermissionToRole(role, Plugins.WHITEBOARD, 'admin')); + permissions?.reader?.forEach(role => this.addPermissionToRole(role, Plugins.WHITEBOARD, 'read')); + permissions?.writer?.forEach(role => this.addPermissionToRole(role, Plugins.WHITEBOARD, 'write')); + }; + private addTranscriptionsPluginToRole = (plugin: TranscriptionPluginPermissions[] = []) => { + for (const transcription of plugin) { + transcription.permissions?.admin?.forEach(role => + this.addPermissionToRole(role, Plugins.TRANSCRIPTIONS, 'admin', transcription.mode), + ); + } + }; + private handleNoiseCancellationPlugin = (plugin?: NoiseCancellationPlugin) => { + if (!this.room) { + return; + } + // it will be called again after internalConnect room initialization, even after network disconnection + this.room.isNoiseCancellationEnabled = !!plugin?.enabled && !!this.room.isNoiseCancellationEnabled; + }; + private setEnv() { const endPoint = this.config?.initEndpoint!; const url = endPoint.split('https://')[1]; diff --git a/packages/hms-video-store/src/selectors/selectors.ts b/packages/hms-video-store/src/selectors/selectors.ts index aeb546e6ea..7de9ef82c9 100644 --- a/packages/hms-video-store/src/selectors/selectors.ts +++ b/packages/hms-video-store/src/selectors/selectors.ts @@ -9,7 +9,7 @@ import { isVideoPlaylist, } from './selectorUtils'; // noinspection ES6PreferShortImport -import { HMSRole, HMSWhiteboard } from '../internal'; +import { HMSRole, HMSTranscriptionMode, HMSTranscriptionState, HMSWhiteboard } from '../internal'; import { HMSException, HMSMessage, @@ -63,7 +63,7 @@ export const selectTracksMap = (store: HMSStore) => store.tracks; /** * Select your media settings - * i.e., choosen audio input device, audio output device and video input device. + * i.e., choosen audio input device, audio output device and video input device, audio mode * @param store */ export const selectLocalMediaSettings = (store: HMSStore) => store.settings; @@ -396,6 +396,16 @@ export const selectIsInPreview = createSelector(selectRoomState, roomState => ro export const selectRoomStarted = createSelector(selectRoom, room => room.roomState !== HMSRoomState.Disconnected); +export const selectIsTranscriptionEnabled = createSelector(selectRoom, room => { + if (!room.transcriptions || room.transcriptions.length <= 0) { + return false; + } + return room.transcriptions.some( + transcription => + transcription.mode === HMSTranscriptionMode.CAPTION && transcription.state === HMSTranscriptionState.STARTED, + ); +}); + /** * Select available roles in the room as a map between the role name and {@link HMSRole} object. */ @@ -442,10 +452,12 @@ export const selectPermissions = createSelector(selectLocalPeerRole, role => rol export const selectRecordingState = createSelector(selectRoom, room => room.recording); export const selectRTMPState = createSelector(selectRoom, room => room.rtmp); export const selectHLSState = createSelector(selectRoom, room => room.hls); +export const selectTranscriptionsState = createSelector(selectRoom, room => room.transcriptions); export const selectSessionId = createSelector(selectRoom, room => room.sessionId); export const selectRoomStartTime = createSelector(selectRoom, room => room.startedAt); export const selectIsLargeRoom = createSelector(selectRoom, room => !!room.isLargeRoom); export const selectIsEffectsEnabled = createSelector(selectRoom, room => !!room.isEffectsEnabled); +export const selectIsVBEnabled = createSelector(selectRoom, room => !!room.isVBEnabled); export const selectEffectsKey = createSelector(selectRoom, room => room.effectsKey); export const selectTemplateAppData = (store: HMSStore) => store.templateAppData; diff --git a/packages/hms-video-store/src/selectors/selectorsByID.ts b/packages/hms-video-store/src/selectors/selectorsByID.ts index 88f123e27c..ecb943d5b5 100644 --- a/packages/hms-video-store/src/selectors/selectorsByID.ts +++ b/packages/hms-video-store/src/selectors/selectorsByID.ts @@ -4,6 +4,7 @@ import { selectFullAppData, selectHMSMessages, selectLocalPeerID, + selectLocalPeerRole, selectMessagesMap, selectPeers, selectPeersMap, @@ -19,6 +20,7 @@ import { isVideoPlaylist, } from './selectorUtils'; import { HMSLogger } from '../common/ui-logger'; +import { HMSTranscriptionMode } from '../internal'; import { HMSAudioTrack, HMSGenericTypes, @@ -163,6 +165,8 @@ export const selectAppDataByPath = (...keys: string[]) => */ export const selectPeerNameByID = byIDCurry(createSelector(selectPeerByIDBare, peer => peer?.name)); +export const selectPeerTypeByID = byIDCurry(createSelector(selectPeerByIDBare, peer => peer?.type)); + /** * Select the {@link HMSTrack} object given a track ID. */ @@ -527,3 +531,12 @@ export const selectMessageByMessageID = (id: string) => createSelector(selectMessagesMap, messages => { return messages[id]; }); + +export const selectIsTranscriptionAllowedByMode = (mode: HMSTranscriptionMode) => + createSelector(selectLocalPeerRole, role => { + if (!role?.permissions.transcriptions) { + return false; + } + // only one admin permission + return role.permissions.transcriptions[mode].length > 0; + }); diff --git a/packages/hms-video-store/src/session-store/interactivity-center/HMSInteractivityCenter.ts b/packages/hms-video-store/src/session-store/interactivity-center/HMSInteractivityCenter.ts index 4e4d93a5ed..20caa323ba 100644 --- a/packages/hms-video-store/src/session-store/interactivity-center/HMSInteractivityCenter.ts +++ b/packages/hms-video-store/src/session-store/interactivity-center/HMSInteractivityCenter.ts @@ -10,6 +10,7 @@ import { HMSPollCreateParams, HMSPollQuestionAnswer, HMSPollQuestionOption, + HMSPollQuestionResponse, HMSPollQuestionResponseCreateParams, HMSPollQuestionType, HMSPollStates, @@ -128,9 +129,10 @@ export class InteractivityCenter implements HMSInteractivityCenter { throw new Error('Invalid poll ID - Poll not found'); } - const canReadPolls = this.store.getLocalPeer()?.role?.permissions.pollRead || false; + const localPeerPermissions = this.store.getLocalPeer()?.role?.permissions; + const canViewSummary = !!(localPeerPermissions?.pollRead || localPeerPermissions?.pollWrite); - if (poll.anonymous || poll.state !== HMSPollStates.STOPPED || !canReadPolls) { + if (poll.anonymous || poll.state !== HMSPollStates.STOPPED || !canViewSummary) { return { entries: [], hasNext: false }; } const pollLeaderboard = await this.transport.signal.fetchPollLeaderboard({ @@ -161,6 +163,39 @@ export class InteractivityCenter implements HMSInteractivityCenter { return { entries: leaderboardEntries, hasNext: !pollLeaderboard.last, summary }; } + async getPollResponses(poll: HMSPoll, self: boolean) { + const serverResponseParams = await this.transport.signal.getPollResponses({ + poll_id: poll.id, + index: 0, + count: 50, + self, + }); + const pollCopy = JSON.parse(JSON.stringify(poll)); + serverResponseParams.responses?.forEach(({ response, peer, final }) => { + const question = poll?.questions?.find(question => question.index === response.question); + if (question) { + const pollResponse: HMSPollQuestionResponse = { + id: response.response_id, + questionIndex: response.question, + option: response.option, + options: response.options, + text: response.text, + responseFinal: final, + peer: { peerid: peer.peerid, userHash: peer.hash, userid: peer.userid, username: peer.username }, + skipped: response.skipped, + type: response.type, + update: response.update, + }; + const existingResponses = question.responses && !self ? [...question.responses] : []; + + if (pollCopy.questions?.[response.question - 1]) { + pollCopy.questions[response.question - 1].responses = [...existingResponses, pollResponse]; + } + } + }); + this.store.setPoll(pollCopy); + this.listener?.onPollsUpdate(HMSPollsUpdate.POLL_STATS_UPDATED, [pollCopy]); + } async getPolls(): Promise { const launchedPollsList = await this.transport.signal.getPollsList({ count: 50, state: 'started' }); const polls: HMSPoll[] = []; @@ -195,10 +230,23 @@ export class InteractivityCenter implements HMSInteractivityCenter { return polls; } + // eslint-disable-next-line complexity private createQuestionSetParams(questionParams: HMSPollQuestionCreateParams, index: number): PollQuestionParams { + // early return if the question has been saved before in a draft + if (questionParams.index) { + const optionsWithIndex = questionParams.options?.map((option, index) => { + return { ...option, index: index + 1 }; + }); + return { + question: { ...questionParams, index: index + 1 }, + options: optionsWithIndex, + answer: questionParams.answer, + }; + } const question: PollQuestionParams['question'] = { ...questionParams, index: index + 1 }; let options: HMSPollQuestionOption[] | undefined; const answer: HMSPollQuestionAnswer = questionParams.answer || { hidden: false }; + if ( Array.isArray(questionParams.options) && [HMSPollQuestionType.SINGLE_CHOICE, HMSPollQuestionType.MULTIPLE_CHOICE].includes(questionParams.type) @@ -209,7 +257,6 @@ export class InteractivityCenter implements HMSInteractivityCenter { weight: option.weight, })); - delete answer?.text; if (questionParams.type === HMSPollQuestionType.SINGLE_CHOICE) { answer.option = questionParams.options.findIndex(option => option.isCorrectAnswer) + 1 || undefined; } else { diff --git a/packages/hms-video-store/src/session-store/interactivity-center/HMSWhiteboardCenter.ts b/packages/hms-video-store/src/session-store/interactivity-center/HMSWhiteboardCenter.ts index 746c26ba1c..3fc7479e89 100644 --- a/packages/hms-video-store/src/session-store/interactivity-center/HMSWhiteboardCenter.ts +++ b/packages/hms-video-store/src/session-store/interactivity-center/HMSWhiteboardCenter.ts @@ -5,6 +5,7 @@ import { InitFlags } from '../../signal/init/models'; import { HMSWhiteboardCreateOptions } from '../../signal/interfaces'; import HMSTransport from '../../transport'; import HMSLogger from '../../utils/logger'; +import { constructWhiteboardURL } from '../../utils/whiteboard'; export class WhiteboardInteractivityCenter implements HMSWhiteboardInteractivityCenter { private TAG = '[HMSWhiteboardInteractivityCenter]'; @@ -40,6 +41,7 @@ export class WhiteboardInteractivityCenter implements HMSWhiteboardInteractivity title: createOptions?.title, attributes: createOptions?.attributes, id: response.id, + url: constructWhiteboardURL(response.token, response.addr, this.store.getEnv()), token: response.token, addr: response.addr, owner: response.owner, @@ -70,6 +72,34 @@ export class WhiteboardInteractivityCenter implements HMSWhiteboardInteractivity this.listener = listener; } + async handleLocalRoleUpdate() { + const whiteboards = this.store.getWhiteboards(); + + for (const whiteboard of whiteboards.values()) { + if (whiteboard.url) { + const response = await this.transport.signal.getWhiteboard({ id: whiteboard.id }); + const localPeer = this.store.getLocalPeer(); + const isOwner = localPeer?.customerUserId === response.owner; + const open = isOwner + ? localPeer.role?.permissions.whiteboard?.includes('admin') + : response.permissions.length > 0; + const newWhiteboard: HMSWhiteboard = { + ...whiteboard, + id: response.id, + url: constructWhiteboardURL(response.token, response.addr, this.store.getEnv()), + token: response.token, + addr: response.addr, + owner: response.owner, + permissions: response.permissions, + open, + }; + + this.store.setWhiteboard(newWhiteboard); + this.listener?.onWhiteboardUpdate(newWhiteboard); + } + } + } + private getCreateOptionsWithDefaults(createOptions?: HMSWhiteboardCreateOptions): HMSWhiteboardCreateOptions { const roles = Object.values(this.store.getKnownRoles()); const reader: Array = []; diff --git a/packages/hms-video-store/src/signal/init/index.ts b/packages/hms-video-store/src/signal/init/index.ts index e48260e498..50d4cfc874 100644 --- a/packages/hms-video-store/src/signal/init/index.ts +++ b/packages/hms-video-store/src/signal/init/index.ts @@ -1,6 +1,9 @@ import { InitConfig } from './models'; import { ErrorFactory } from '../../error/ErrorFactory'; import { HMSAction } from '../../error/HMSAction'; +import { HMSICEServer } from '../../interfaces'; +import { HMSException } from '../../internal'; +import { transformIceServerConfig } from '../../utils/ice-server-config'; import HMSLogger from '../../utils/logger'; const TAG = '[InitService]'; @@ -26,30 +29,32 @@ export default class InitService { userAgent, initEndpoint = 'https://prod-init.100ms.live', region = '', + iceServers, }: { token: string; peerId: string; userAgent: string; initEndpoint?: string; region?: string; + iceServers?: HMSICEServer[]; }): Promise { HMSLogger.d(TAG, `fetchInitConfig: initEndpoint=${initEndpoint} token=${token} peerId=${peerId} region=${region} `); const url = getUrl(initEndpoint, peerId, userAgent, region); try { const response = await fetch(url, { - headers: { - Authorization: `Bearer ${token}`, - }, + headers: { Authorization: `Bearer ${token}` }, }); try { const config = await response.clone().json(); this.handleError(response, config); HMSLogger.d(TAG, `config is ${JSON.stringify(config, null, 2)}`); - return transformInitConfig(config); + return transformInitConfig(config, iceServers); } catch (err) { const text = await response.text(); HMSLogger.e(TAG, 'json error', (err as Error).message, text); - throw ErrorFactory.APIErrors.ServerErrors(response.status, HMSAction.INIT, text); + throw err instanceof HMSException + ? err + : ErrorFactory.APIErrors.ServerErrors(response.status, HMSAction.INIT, (err as Error).message); } } catch (err) { const error = err as Error; @@ -78,9 +83,12 @@ export function getUrl(endpoint: string, peerId: string, userAgent: string, regi } } -export function transformInitConfig(config: any): InitConfig { +export function transformInitConfig(config: any, iceServers?: HMSICEServer[]): InitConfig { return { ...config, - rtcConfiguration: { ...config.rtcConfiguration, iceServers: config.rtcConfiguration?.ice_servers }, + rtcConfiguration: { + ...config.rtcConfiguration, + iceServers: transformIceServerConfig(config.rtcConfiguration?.ice_servers, iceServers), + }, }; } diff --git a/packages/hms-video-store/src/signal/init/models.ts b/packages/hms-video-store/src/signal/init/models.ts index 8e4657cdd9..918c4f80e1 100644 --- a/packages/hms-video-store/src/signal/init/models.ts +++ b/packages/hms-video-store/src/signal/init/models.ts @@ -58,4 +58,9 @@ export enum InitFlags { FLAG_DISABLE_VIDEO_TRACK_AUTO_UNSUBSCRIBE = 'disableVideoTrackAutoUnsubscribe', FLAG_WHITEBOARD_ENABLED = 'whiteboardEnabled', FLAG_EFFECTS_SDK_ENABLED = 'effectsSDKEnabled', + FLAG_VB_ENABLED = 'vb', + FLAG_HIPAA_ENABLED = 'hipaa', + FLAG_NOISE_CANCELLATION = 'noiseCancellation', + FLAG_SCALE_SCREENSHARE_BASED_ON_PIXELS = 'scaleScreenshareBasedOnPixels', + FLAG_DISABLE_NONE_LAYER_REQUEST = 'disableNoneLayerRequest', } diff --git a/packages/hms-video-store/src/signal/interfaces/responses.ts b/packages/hms-video-store/src/signal/interfaces/responses.ts index 67998885ab..d4af4c1b09 100644 --- a/packages/hms-video-store/src/signal/interfaces/responses.ts +++ b/packages/hms-video-store/src/signal/interfaces/responses.ts @@ -1,5 +1,5 @@ import { HMSPermissionType } from '../../interfaces'; -import { PeerNotificationInfo } from '../../notification-manager'; +import { FindPeerByNameInfo, PeerNotificationInfo } from '../../notification-manager'; export interface BroadcastResponse { timestamp: number; @@ -32,6 +32,14 @@ export interface PeersIterationResponse { peers: PeerNotificationInfo[]; } +export interface FindPeerByNameResponse { + count: number; + limit: number; + offset: number; + eof: boolean; + peers: FindPeerByNameInfo[]; +} + export interface CreateWhiteboardResponse { id: string; owner: string; diff --git a/packages/hms-video-store/src/signal/interfaces/superpowers.ts b/packages/hms-video-store/src/signal/interfaces/superpowers.ts index 6b159b7ba6..3a55a0e6d1 100644 --- a/packages/hms-video-store/src/signal/interfaces/superpowers.ts +++ b/packages/hms-video-store/src/signal/interfaces/superpowers.ts @@ -1,4 +1,4 @@ -import { HMSTrackSource } from '../..'; +import { HMSTrackSource, HMSTranscriptionMode } from '../..'; import { HLSTimedMetadata, RTMPRecordingResolution } from '../../interfaces'; /** @@ -57,6 +57,9 @@ export interface UpdatePeerRequestParams { data?: string; } +export interface StartTranscriptionRequestParams { + mode: HMSTranscriptionMode; +} export interface SetSessionMetadataParams { key?: string; data: any; @@ -81,18 +84,24 @@ export interface HLSVariant { metadata?: string; } -export interface getPeerRequestParams { - peerId: string; +export interface GetPeerRequestParams { + peer_id: string; } -export interface findPeersRequestParams { +export interface FindPeersRequestParams { peers?: string[]; role?: string; group?: string; limit: number; } -export interface peerIterRequestParams { +export interface FindPeerByNameRequestParams { + query?: string; + limit?: number; + offset?: number; +} + +export interface PeerIterRequestParams { iterator: string; limit: number; } diff --git a/packages/hms-video-store/src/signal/jsonrpc/index.ts b/packages/hms-video-store/src/signal/jsonrpc/index.ts index 9fa76719f5..51e357ce56 100644 --- a/packages/hms-video-store/src/signal/jsonrpc/index.ts +++ b/packages/hms-video-store/src/signal/jsonrpc/index.ts @@ -5,7 +5,7 @@ import { HMSConnectionRole, HMSTrickle } from '../../connection/model'; import { ErrorFactory } from '../../error/ErrorFactory'; import { HMSAction } from '../../error/HMSAction'; import { HMSException } from '../../error/HMSException'; -import { SendMessage } from '../../notification-manager'; +import { PeerNotificationInfo, SendMessage } from '../../notification-manager'; import { DEFAULT_SIGNAL_PING_INTERVAL, DEFAULT_SIGNAL_PING_TIMEOUT, @@ -20,8 +20,10 @@ import { AcceptRoleChangeParams, BroadcastResponse, CreateWhiteboardResponse, - findPeersRequestParams, - getPeerRequestParams, + FindPeerByNameRequestParams, + FindPeerByNameResponse, + FindPeersRequestParams, + GetPeerRequestParams, GetSessionMetadataResponse, GetWhiteboardResponse, HLSRequestParams, @@ -30,7 +32,7 @@ import { HMSWhiteboardCreateOptions, JoinLeaveGroupResponse, MultiTrackUpdateRequestParams, - peerIterRequestParams, + PeerIterRequestParams, PeersIterationResponse, PollInfoGetParams, PollInfoGetResponse, @@ -60,6 +62,7 @@ import { SetSessionMetadataParams, SetSessionMetadataResponse, StartRTMPOrRecordingRequestParams, + StartTranscriptionRequestParams, Track, TrackUpdateRequestParams, UpdatePeerRequestParams, @@ -88,6 +91,7 @@ export default class JsonRpcSignal { private _isConnected = false; private id = 0; + private sfuNodeId: string | undefined; private onCloseHandler: (event: CloseEvent) => void = () => {}; @@ -95,6 +99,10 @@ export default class JsonRpcSignal { return this._isConnected; } + setSfuNodeId(sfuNodeId?: string) { + this.sfuNodeId = sfuNodeId; + } + public setIsConnected(newValue: boolean, reason = '') { HMSLogger.d(this.TAG, `isConnected set id: ${this.id}, oldValue: ${this._isConnected}, newValue: ${newValue}`); if (this._isConnected === newValue) { @@ -241,7 +249,7 @@ export default class JsonRpcSignal { simulcast: boolean, onDemandTracks: boolean, offer?: RTCSessionDescriptionInit, - ): Promise { + ): Promise { if (!this.isConnected) { throw ErrorFactory.WebSocketConnectionErrors.WebSocketConnectionLost( HMSAction.JOIN, @@ -257,7 +265,10 @@ export default class JsonRpcSignal { simulcast, onDemandTracks, }; - const response: RTCSessionDescriptionInit = await this.internalCall(HMSSignalMethod.JOIN, params); + const response: RTCSessionDescriptionInit & { sfu_node_id: string | undefined } = await this.internalCall( + HMSSignalMethod.JOIN, + params, + ); this.isJoinCompleted = true; this.pendingTrickle.forEach(({ target, candidate }) => this.trickle(target, candidate)); @@ -269,7 +280,7 @@ export default class JsonRpcSignal { trickle(target: HMSConnectionRole, candidate: RTCIceCandidateInit) { if (this.isJoinCompleted) { - this.notify(HMSSignalMethod.TRICKLE, { target, candidate }); + this.notify(HMSSignalMethod.TRICKLE, { target, candidate, sfu_node_id: this.sfuNodeId }); } else { this.pendingTrickle.push({ target, candidate }); } @@ -279,12 +290,13 @@ export default class JsonRpcSignal { const response = await this.call(HMSSignalMethod.OFFER, { desc, tracks: Object.fromEntries(tracks), + sfu_node_id: this.sfuNodeId, }); return response as RTCSessionDescriptionInit; } answer(desc: RTCSessionDescriptionInit) { - this.notify(HMSSignalMethod.ANSWER, { desc }); + this.notify(HMSSignalMethod.ANSWER, { desc, sfu_node_id: this.sfuNodeId }); } trackUpdate(tracks: Map) { @@ -364,6 +376,14 @@ export default class JsonRpcSignal { await this.call(HMSSignalMethod.STOP_HLS_STREAMING, { ...params }); } + async startTranscription(params: StartTranscriptionRequestParams) { + await this.call(HMSSignalMethod.START_TRANSCRIPTION, { ...params }); + } + + async stopTranscription(params: StartTranscriptionRequestParams) { + await this.call(HMSSignalMethod.STOP_TRANSCRIPTION, { ...params }); + } + async sendHLSTimedMetadata(params?: HLSTimedMetadataParams): Promise { await this.call(HMSSignalMethod.HLS_TIMED_METADATA, { ...params }); } @@ -372,8 +392,8 @@ export default class JsonRpcSignal { await this.call(HMSSignalMethod.UPDATE_PEER_METADATA, { ...params }); } - async getPeer(params: getPeerRequestParams) { - await this.call(HMSSignalMethod.GET_PEER, { ...params }); + async getPeer(params: GetPeerRequestParams): Promise { + return await this.call(HMSSignalMethod.GET_PEER, { ...params }); } async joinGroup(name: string): Promise { @@ -392,14 +412,18 @@ export default class JsonRpcSignal { await this.call(HMSSignalMethod.GROUP_REMOVE, { name, peer_id: peerId }); } - async peerIterNext(params: peerIterRequestParams): Promise { + async peerIterNext(params: PeerIterRequestParams): Promise { return await this.call(HMSSignalMethod.PEER_ITER_NEXT, params); } - async findPeers(params: findPeersRequestParams): Promise { + async findPeers(params: FindPeersRequestParams): Promise { return await this.call(HMSSignalMethod.FIND_PEER, params); } + async findPeerByName(params: FindPeerByNameRequestParams): Promise { + return await this.call(HMSSignalMethod.SEARCH_BY_NAME, params); + } + setSessionMetadata(params: SetSessionMetadataParams) { if (!this.isConnected) { throw ErrorFactory.WebSocketConnectionErrors.WebSocketConnectionLost( @@ -595,16 +619,17 @@ export default class JsonRpcSignal { private async call(method: HMSSignalMethod, params: Record): Promise { const MAX_RETRIES = 3; let error: HMSException = ErrorFactory.WebsocketMethodErrors.ServerErrors(500, method, `Default ${method} error`); - this.validateConnection(); let retry; for (retry = 1; retry <= MAX_RETRIES; retry++) { try { + this.validateConnection(); HMSLogger.d(this.TAG, `Try number ${retry} sending ${method}`, params); return await this.internalCall(method, params); } catch (err) { error = err as HMSException; HMSLogger.e(this.TAG, `Failed sending ${method} try: ${retry}`, { method, params, error }); - const shouldRetry = parseInt(`${error.code / 100}`) === 5 || error.code === 429; + // 1003 is websocket disconnect - could be because you are offline - retry with delay in this case as well + const shouldRetry = parseInt(`${error.code / 100}`) === 5 || error.code === 429 || error.code === 1003; if (!shouldRetry) { break; } diff --git a/packages/hms-video-store/src/signal/jsonrpc/models.ts b/packages/hms-video-store/src/signal/jsonrpc/models.ts index 2344ec3e70..d3aa42219a 100644 --- a/packages/hms-video-store/src/signal/jsonrpc/models.ts +++ b/packages/hms-video-store/src/signal/jsonrpc/models.ts @@ -39,6 +39,8 @@ export enum HMSSignalMethod { UPDATE_PEER_METADATA = 'peer-update', START_HLS_STREAMING = 'hls-start', STOP_HLS_STREAMING = 'hls-stop', + START_TRANSCRIPTION = 'transcription-start', + STOP_TRANSCRIPTION = 'transcription-stop', HLS_TIMED_METADATA = 'hls-timed-metadata', SET_METADATA = 'set-metadata', GET_METADATA = 'get-metadata', @@ -56,6 +58,7 @@ export enum HMSSignalMethod { POLL_LEADERBOARD = 'poll-leaderboard', GET_PEER = 'get-peer', FIND_PEER = 'find-peer', + SEARCH_BY_NAME = 'peer-name-search', PEER_ITER_NEXT = 'peer-iter-next', GROUP_JOIN = 'group-join', GROUP_LEAVE = 'group-leave', diff --git a/packages/hms-video-store/src/test/fakeStore/fakeHMSStore.ts b/packages/hms-video-store/src/test/fakeStore/fakeHMSStore.ts index a0aecfc784..a836b3ceab 100644 --- a/packages/hms-video-store/src/test/fakeStore/fakeHMSStore.ts +++ b/packages/hms-video-store/src/test/fakeStore/fakeHMSStore.ts @@ -9,7 +9,7 @@ import { HMSTrackSource, HMSTrackType, } from '../../'; -import { HMSSimulcastLayer } from '../../internal'; +import { HMSPeerType, HMSSimulcastLayer } from '../../internal'; import { HMSAudioTrack, HMSPlaylist, HMSPlaylistType, HMSRole, HMSScreenVideoTrack, HMSVideoTrack } from '../../schema'; function makeTrack( @@ -81,6 +81,7 @@ export const makeFakeStore = (): HMSStore => { variants: [], }, sessionId: '', + transcriptions: [], }, appData: { isAudioOnly: true, @@ -98,6 +99,7 @@ export const makeFakeStore = (): HMSStore => { metadata: '{}', groups: [], isHandRaised: false, + type: HMSPeerType.REGULAR, }, '2': { id: '2', @@ -110,6 +112,7 @@ export const makeFakeStore = (): HMSStore => { metadata: '{"hello":"world"}', groups: [], isHandRaised: false, + type: HMSPeerType.REGULAR, }, '3': { id: '3', @@ -121,15 +124,16 @@ export const makeFakeStore = (): HMSStore => { auxiliaryTracks: [], groups: [], isHandRaised: false, + type: HMSPeerType.REGULAR, }, }, tracks: { - '101': makeTrack('101', 'video', 'regular', '1'), - '102': makeTrack('102', 'audio', 'regular', '1'), - '103': makeTrack('103', 'video', 'regular', '2'), - '104': makeTrack('104', 'audio', 'regular', '2'), + '101': makeTrack('101', 'video', HMSPeerType.REGULAR, '1'), + '102': makeTrack('102', 'audio', HMSPeerType.REGULAR, '1'), + '103': makeTrack('103', 'video', HMSPeerType.REGULAR, '2'), + '104': makeTrack('104', 'audio', HMSPeerType.REGULAR, '2'), '105': makeTrack('105', 'video', 'screen', '2'), - '106': makeTrack('106', 'audio', 'regular', '2'), + '106': makeTrack('106', 'audio', HMSPeerType.REGULAR, '2'), '107': makeTrack('107', 'audio', 'screen', '2'), }, playlist: { diff --git a/packages/hms-video-store/src/test/fixtures.ts b/packages/hms-video-store/src/test/fixtures.ts index dd624ecedd..1db9e7ac2c 100644 --- a/packages/hms-video-store/src/test/fixtures.ts +++ b/packages/hms-video-store/src/test/fixtures.ts @@ -1,3 +1,4 @@ +import { HMSPeerType } from '../interfaces'; import { HMSAudioTrack, HMSPeer, HMSTrackType, HMSVideoTrack } from '../'; let counter = 100; @@ -20,5 +21,6 @@ export const makeFakePeer = (): HMSPeer => { videoTrack: '', groups: [], isHandRaised: false, + type: HMSPeerType.REGULAR, }; }; diff --git a/packages/hms-video-store/src/transport/RetryScheduler.ts b/packages/hms-video-store/src/transport/RetryScheduler.ts index 8695e5a577..7230612a99 100644 --- a/packages/hms-video-store/src/transport/RetryScheduler.ts +++ b/packages/hms-video-store/src/transport/RetryScheduler.ts @@ -1,7 +1,7 @@ import { Dependencies as TFCDependencies, TransportFailureCategory as TFC } from './models/TransportFailureCategory'; import { TransportState } from './models/TransportState'; import { HMSException } from '../error/HMSException'; -import { MAX_TRANSPORT_RETRIES, MAX_TRANSPORT_RETRY_DELAY } from '../utils/constants'; +import { MAX_TRANSPORT_RETRY_TIME } from '../utils/constants'; import HMSLogger from '../utils/logger'; import { PromiseWithCallbacks } from '../utils/promise'; @@ -23,7 +23,7 @@ interface ScheduleTaskParams { error: HMSException; task: RetryTask; originalState: TransportState; - maxFailedRetries?: number; + maxRetryTime?: number; changeState?: boolean; } @@ -42,10 +42,10 @@ export class RetryScheduler { error, task, originalState, - maxFailedRetries = MAX_TRANSPORT_RETRIES, + maxRetryTime = MAX_TRANSPORT_RETRY_TIME, changeState = true, }: ScheduleTaskParams) { - await this.scheduleTask({ category, error, changeState, task, originalState, maxFailedRetries }); + await this.scheduleTask({ category, error, changeState, task, originalState, maxRetryTime, failedAt: Date.now() }); } reset() { @@ -65,9 +65,10 @@ export class RetryScheduler { changeState, task, originalState, - maxFailedRetries = MAX_TRANSPORT_RETRIES, + failedAt, + maxRetryTime = MAX_TRANSPORT_RETRY_TIME, failedRetryCount = 0, - }: ScheduleTaskParams & { failedRetryCount?: number }): Promise { + }: ScheduleTaskParams & { failedAt: number; failedRetryCount?: number }): Promise { HMSLogger.d(this.TAG, 'schedule: ', { category: TFC[category], error }); // First schedule call @@ -113,16 +114,7 @@ export class RetryScheduler { } } - if (failedRetryCount >= maxFailedRetries || hasFailedDependency) { - error.description += `. [${TFC[category]}] Could not recover after ${failedRetryCount} tries`; - - if (hasFailedDependency) { - error.description += ` Could not recover all of it's required dependencies - [${(dependencies as Array) - .map(dep => TFC[dep]) - .toString()}]`; - } - error.isTerminal = true; - + const handleTerminalError = (error: HMSException) => { // @NOTE: Don't reject to throw error for dependencies, use onStateChange // const taskPromise = this.inProgress.get(category); this.inProgress.delete(category); @@ -138,13 +130,26 @@ export class RetryScheduler { } return; + }; + + const timeElapsedSinceError = Date.now() - failedAt; + if (timeElapsedSinceError >= maxRetryTime || hasFailedDependency) { + error.description += `. [${TFC[category]}] Could not recover after ${timeElapsedSinceError} milliseconds`; + + if (hasFailedDependency) { + error.description += ` Could not recover all of it's required dependencies - [${(dependencies as Array) + .map(dep => TFC[dep]) + .toString()}]`; + } + error.isTerminal = true; + return handleTerminalError(error); } if (changeState) { this.onStateChange(TransportState.Reconnecting, error); } - const delay = this.getDelayForRetryCount(category, failedRetryCount); + const delay = this.getDelayForRetryCount(category); HMSLogger.d( this.TAG, @@ -156,11 +161,18 @@ export class RetryScheduler { taskSucceeded = await this.setTimeoutPromise(task, delay); } catch (ex) { taskSucceeded = false; - HMSLogger.w( - this.TAG, - `[${TFC[category]}] Un-caught exception ${(ex as HMSException).name} in retry-task, initiating retry`, - ex, - ); + const error = ex as HMSException; + + if (error.isTerminal) { + HMSLogger.e(this.TAG, `[${TFC[category]}] Un-caught terminal exception ${error.name} in retry-task`, ex); + return handleTerminalError(error); + } else { + HMSLogger.w( + this.TAG, + `[${TFC[category]}] Un-caught exception ${error.name} in retry-task, initiating retry`, + ex, + ); + } } if (taskSucceeded) { @@ -171,7 +183,10 @@ export class RetryScheduler { if (changeState && this.inProgress.size === 0) { this.onStateChange(originalState); } - HMSLogger.d(this.TAG, `schedule: [${TFC[category]}] [failedRetryCount=${failedRetryCount}] Recovered ♻️`); + HMSLogger.d( + this.TAG, + `schedule: [${TFC[category]}] [failedRetryCount=${failedRetryCount}] Recovered ♻️ after ${timeElapsedSinceError}ms`, + ); } else { await this.scheduleTask({ category, @@ -179,25 +194,23 @@ export class RetryScheduler { changeState, task, originalState, - maxFailedRetries, + maxRetryTime, + failedAt, failedRetryCount: failedRetryCount + 1, }); } } - private getBaseDelayForTask(category: TFC, n: number) { + private getDelayForRetryCount(category: TFC) { + const jitter = category === TFC.JoinWSMessageFailed ? Math.random() * 2 : Math.random(); + let delaySeconds = 0; if (category === TFC.JoinWSMessageFailed) { // linear backoff(2 + jitter for every retry) - return 2; + delaySeconds = 2 + jitter; + } else if (category === TFC.SignalDisconnect) { + delaySeconds = 1; } - // exponential backoff - return Math.pow(2, n); - } - - private getDelayForRetryCount(category: TFC, n: number) { - const delay = this.getBaseDelayForTask(category, n); - const jitter = category === TFC.JoinWSMessageFailed ? Math.random() * 2 : Math.random(); - return Math.round(Math.min(delay + jitter, MAX_TRANSPORT_RETRY_DELAY) * 1000); + return delaySeconds * 1000; } private async setTimeoutPromise(task: () => Promise, delay: number): Promise { diff --git a/packages/hms-video-store/src/transport/index.ts b/packages/hms-video-store/src/transport/index.ts index 3bf39031b4..e589e3c6c8 100644 --- a/packages/hms-video-store/src/transport/index.ts +++ b/packages/hms-video-store/src/transport/index.ts @@ -11,18 +11,20 @@ import { AnalyticsTimer, TimedEvent } from '../analytics/AnalyticsTimer'; import { HTTPAnalyticsTransport } from '../analytics/HTTPAnalyticsTransport'; import { SignalAnalyticsTransport } from '../analytics/signal-transport/SignalAnalyticsTransport'; import { PublishStatsAnalytics, SubscribeStatsAnalytics } from '../analytics/stats'; +import { PluginUsageTracker } from '../common/PluginUsageTracker'; import { HMSConnectionRole, HMSTrickle } from '../connection/model'; import { IPublishConnectionObserver } from '../connection/publish/IPublishConnectionObserver'; import HMSPublishConnection from '../connection/publish/publishConnection'; import ISubscribeConnectionObserver from '../connection/subscribe/ISubscribeConnectionObserver'; import HMSSubscribeConnection from '../connection/subscribe/subscribeConnection'; import { DeviceManager } from '../device-manager'; +import { HMSDiagnosticsConnectivityListener } from '../diagnostics/interfaces'; import { ErrorCodes } from '../error/ErrorCodes'; import { ErrorFactory } from '../error/ErrorFactory'; import { HMSAction } from '../error/HMSAction'; import { HMSException } from '../error/HMSException'; import { EventBus } from '../events/EventBus'; -import { HMSRole } from '../interfaces'; +import { HMSICEServer, HMSRole, HMSTrackUpdate, HMSUpdateListener } from '../interfaces'; import { HMSLocalStream } from '../media/streams/HMSLocalStream'; import { HMSLocalTrack, HMSLocalVideoTrack, HMSTrack } from '../media/tracks'; import { TrackState } from '../notification-manager'; @@ -34,7 +36,6 @@ import { ISignalEventsObserver } from '../signal/ISignalEventsObserver'; import JsonRpcSignal from '../signal/jsonrpc'; import { ICE_DISCONNECTION_TIMEOUT, - MAX_TRANSPORT_RETRIES, PROTOCOL_SPEC, PROTOCOL_VERSION, PUBLISH_STATS_PUSH_INTERVAL, @@ -77,7 +78,13 @@ export default class HMSTransport { private publishStatsAnalytics?: PublishStatsAnalytics; private subscribeStatsAnalytics?: SubscribeStatsAnalytics; private maxSubscribeBitrate = 0; + private connectivityListener?: HMSDiagnosticsConnectivityListener; + private sfuNodeId?: string; joinRetryCount = 0; + private publishDisconnectTimer = 0; + private listener?: HMSUpdateListener; + private onScreenshareStop = () => {}; + private screenStream = new Set(); constructor( private observer: ITransportObserver, @@ -86,13 +93,9 @@ export default class HMSTransport { private eventBus: EventBus, private analyticsEventsService: AnalyticsEventsService, private analyticsTimer: AnalyticsTimer, + private pluginUsageTracker: PluginUsageTracker, ) { - this.webrtcInternals = new HMSWebrtcInternals( - this.store, - this.eventBus, - this.publishConnection?.nativeConnection, - this.subscribeConnection?.nativeConnection, - ); + this.webrtcInternals = new HMSWebrtcInternals(this.store, this.eventBus); const onStateChange = async (state: TransportState, error?: HMSException) => { if (state !== this.state) { @@ -107,8 +110,8 @@ export default class HMSTransport { this.maxSubscribeBitrate = Math.max(this.maxSubscribeBitrate, currentSubscribeBitrate); }); - this.eventBus.localAudioEnabled.subscribe(({ track }) => this.trackUpdate(track)); - this.eventBus.localVideoEnabled.subscribe(({ track }) => this.trackUpdate(track)); + this.eventBus.localAudioEnabled.subscribe(({ track, enabled }) => this.trackUpdate(track, enabled)); + this.eventBus.localVideoEnabled.subscribe(({ track, enabled }) => this.trackUpdate(track, enabled)); } /** @@ -118,12 +121,36 @@ export default class HMSTransport { */ private readonly callbacks = new Map(); + setListener = (listener: HMSUpdateListener) => { + this.listener = listener; + }; + + setOnScreenshareStop = (onStop: () => void) => { + this.onScreenshareStop = onStop; + }; + + getWebsocketEndpoint(): string | undefined { + if (!this.initConfig) { + return; + } + return this.initConfig.endpoint; + } + private signalObserver: ISignalEventsObserver = { - onOffer: async (jsep: RTCSessionDescriptionInit) => { + // eslint-disable-next-line complexity + onOffer: async (jsep: RTCSessionDescriptionInit & { sfu_node_id?: string }) => { try { if (!this.subscribeConnection) { return; } + if ( + jsep.sfu_node_id && + this.subscribeConnection.sfuNodeId && + this.subscribeConnection.sfuNodeId !== jsep.sfu_node_id + ) { + HMSLogger.d(TAG, 'ignoring old offer'); + return; + } await this.subscribeConnection.setRemoteDescription(jsep); HMSLogger.d( TAG, @@ -145,7 +172,7 @@ export default class HMSTransport { if (err instanceof HMSException) { ex = err; } else { - ex = ErrorFactory.GenericErrors.Unknown(HMSAction.PUBLISH, (err as Error).message); + ex = ErrorFactory.GenericErrors.Unknown(HMSAction.SUBSCRIBE, (err as Error).message); } this.observer.onFailure(ex); this.eventBus.analytics.publish(AnalyticsEventFactory.subscribeFail(ex)); @@ -214,171 +241,6 @@ export default class HMSTransport { private publishDtlsStateTimer = 0; private lastPublishDtlsState: RTCDtlsTransportState = 'new'; - private publishConnectionObserver: IPublishConnectionObserver = { - onRenegotiationNeeded: async () => { - await this.performPublishRenegotiation(); - }, - - // eslint-disable-next-line complexity - onDTLSTransportStateChange: (state?: RTCDtlsTransportState) => { - const log = state === 'failed' ? HMSLogger.w.bind(HMSLogger) : HMSLogger.d.bind(HMSLogger); - log(TAG, `Publisher on dtls transport state change: ${state}`); - - if (!state || this.lastPublishDtlsState === state) { - return; - } - - this.lastPublishDtlsState = state; - if (this.publishDtlsStateTimer !== 0) { - clearTimeout(this.publishDtlsStateTimer); - this.publishDtlsStateTimer = 0; - } - - if (state !== 'connecting' && state !== 'failed') { - return; - } - - const timeout = this.initConfig?.config?.dtlsStateTimeouts?.[state]; - if (!timeout || timeout <= 0) { - return; - } - - // if we're in connecting check again after timeout - // hotfix: mitigate https://100ms.atlassian.net/browse/LIVE-1924 - this.publishDtlsStateTimer = window.setTimeout(() => { - const newState = this.publishConnection?.nativeConnection.connectionState; - if (newState && state && newState === state) { - // stuck in either `connecting` or `failed` state for long time - const err = ErrorFactory.WebrtcErrors.ICEFailure( - HMSAction.PUBLISH, - `DTLS transport state ${state} timeout:${timeout}ms`, - true, - ); - this.eventBus.analytics.publish(AnalyticsEventFactory.disconnect(err)); - this.observer.onFailure(err); - } - }, timeout); - }, - - onDTLSTransportError: (error: Error) => { - HMSLogger.e(TAG, `onDTLSTransportError ${error.name} ${error.message}`, error); - this.eventBus.analytics.publish(AnalyticsEventFactory.disconnect(error)); - }, - - onIceConnectionChange: async (newState: RTCIceConnectionState) => { - const log = newState === 'disconnected' ? HMSLogger.w.bind(HMSLogger) : HMSLogger.d.bind(HMSLogger); - log(TAG, `Publish ice connection state change: ${newState}`); - - // @TODO: Uncomment this and remove connectionstatechange - if (newState === 'failed') { - // await this.handleIceConnectionFailure(HMSConnectionRole.Publish); - } - }, - - // @TODO(eswar): Remove this. Use iceconnectionstate change with interval and threshold. - onConnectionStateChange: async (newState: RTCPeerConnectionState) => { - const log = newState === 'disconnected' ? HMSLogger.w.bind(HMSLogger) : HMSLogger.d.bind(HMSLogger); - log(TAG, `Publish connection state change: ${newState}`); - - if (newState === 'connected') { - this.publishConnection?.logSelectedIceCandidatePairs(); - } - - if (newState === 'disconnected') { - // if state stays disconnected for 5 seconds, retry - setTimeout(() => { - if (this.publishConnection?.connectionState === 'disconnected') { - this.handleIceConnectionFailure( - HMSConnectionRole.Publish, - ErrorFactory.WebrtcErrors.ICEDisconnected( - HMSAction.PUBLISH, - `local candidate - ${this.publishConnection?.selectedCandidatePair?.local.candidate}; remote candidate - ${this.publishConnection?.selectedCandidatePair?.remote.candidate}`, - ), - ); - } - }, ICE_DISCONNECTION_TIMEOUT); - } - - if (newState === 'failed') { - await this.handleIceConnectionFailure( - HMSConnectionRole.Publish, - ErrorFactory.WebrtcErrors.ICEFailure( - HMSAction.PUBLISH, - `local candidate - ${this.publishConnection?.selectedCandidatePair?.local.candidate}; remote candidate - ${this.publishConnection?.selectedCandidatePair?.remote.candidate}`, - ), - ); - } - }, - }; - - private subscribeConnectionObserver: ISubscribeConnectionObserver = { - onApiChannelMessage: (message: string) => { - this.observer.onNotification(JSON.parse(message)); - }, - - onTrackAdd: (track: HMSTrack) => { - HMSLogger.d(TAG, '[Subscribe] onTrackAdd', `${track}`); - this.observer.onTrackAdd(track); - }, - - onTrackRemove: (track: HMSTrack) => { - HMSLogger.d(TAG, '[Subscribe] onTrackRemove', `${track}`); - this.observer.onTrackRemove(track); - }, - - onIceConnectionChange: async (newState: RTCIceConnectionState) => { - const log = newState === 'disconnected' ? HMSLogger.w.bind(HMSLogger) : HMSLogger.d.bind(HMSLogger); - log(TAG, `Subscribe ice connection state change: ${newState}`); - - if (newState === 'failed') { - // await this.handleIceConnectionFailure(HMSConnectionRole.Subscribe); - } - - if (newState === 'connected') { - const callback = this.callbacks.get(SUBSCRIBE_ICE_CONNECTION_CALLBACK_ID); - this.callbacks.delete(SUBSCRIBE_ICE_CONNECTION_CALLBACK_ID); - - if (callback) { - callback.promise.resolve(true); - } - } - }, - - // @TODO(eswar): Remove this. Use iceconnectionstate change with interval and threshold. - onConnectionStateChange: async (newState: RTCPeerConnectionState) => { - const log = newState === 'disconnected' ? HMSLogger.w.bind(HMSLogger) : HMSLogger.d.bind(HMSLogger); - log(TAG, `Subscribe connection state change: ${newState}`); - - if (newState === 'failed') { - await this.handleIceConnectionFailure( - HMSConnectionRole.Subscribe, - ErrorFactory.WebrtcErrors.ICEFailure( - HMSAction.SUBSCRIBE, - `local candidate - ${this.subscribeConnection?.selectedCandidatePair?.local.candidate}; remote candidate - ${this.subscribeConnection?.selectedCandidatePair?.remote.candidate}`, - ), - ); - } - - if (newState === 'disconnected') { - setTimeout(() => { - if (this.subscribeConnection?.connectionState === 'disconnected') { - this.handleIceConnectionFailure( - HMSConnectionRole.Subscribe, - ErrorFactory.WebrtcErrors.ICEDisconnected( - HMSAction.SUBSCRIBE, - `local candidate - ${this.subscribeConnection?.selectedCandidatePair?.local.candidate}; remote candidate - ${this.subscribeConnection?.selectedCandidatePair?.remote.candidate}`, - ), - ); - } - }, ICE_DISCONNECTION_TIMEOUT); - } - - if (newState === 'connected') { - this.handleSubscribeConnectionConnected(); - } - }, - }; - getWebrtcInternals() { return this.webrtcInternals; } @@ -389,14 +251,19 @@ export default class HMSTransport { return flags.includes(flag); } + setConnectivityListener(listener: HMSDiagnosticsConnectivityListener) { + this.connectivityListener = listener; + } + async preview( token: string, endpoint: string, peerId: string, customData: { name: string; metaData: string }, autoSubscribeVideo = false, + iceServers?: HMSICEServer[], ): Promise { - const initConfig = await this.connect(token, endpoint, peerId, customData, autoSubscribeVideo); + const initConfig = await this.connect(token, endpoint, peerId, customData, autoSubscribeVideo, iceServers); this.state = TransportState.Preview; this.observer.onStateChange(this.state); return initConfig; @@ -408,11 +275,12 @@ export default class HMSTransport { customData: { name: string; metaData: string }, initEndpoint: string, autoSubscribeVideo = false, + iceServers?: HMSICEServer[], ): Promise { HMSLogger.d(TAG, 'join: started ⏰'); try { if (!this.signal.isConnected || !this.initConfig) { - await this.connect(authToken, initEndpoint, peerId, customData, autoSubscribeVideo); + await this.connect(authToken, initEndpoint, peerId, customData, autoSubscribeVideo, iceServers); } this.validateNotDisconnected('connect'); @@ -420,7 +288,7 @@ export default class HMSTransport { if (this.initConfig) { await this.waitForLocalRoleAvailability(); await this.createConnectionsAndNegotiateJoin(customData, autoSubscribeVideo); - await this.initRtcStatsMonitor(); + this.initStatsAnalytics(); HMSLogger.d(TAG, '✅ join: Negotiated over PUBLISH connection'); } @@ -445,6 +313,7 @@ export default class HMSTransport { peerId: string, customData: { name: string; metaData: string }, autoSubscribeVideo = false, + iceServers?: HMSICEServer[], ): Promise { this.setTransportStateForConnect(); this.joinParameters = new JoinParameters( @@ -454,9 +323,10 @@ export default class HMSTransport { customData.metaData, endpoint, autoSubscribeVideo, + iceServers, ); try { - const response = await this.internalConnect(token, endpoint, peerId); + const response = await this.internalConnect(token, endpoint, peerId, iceServers); return response; } catch (error) { const shouldRetry = @@ -472,7 +342,7 @@ export default class HMSTransport { if (shouldRetry) { const task = async () => { - await this.internalConnect(token, endpoint, peerId); + await this.internalConnect(token, endpoint, peerId, iceServers); return Boolean(this.initConfig && this.initConfig.endpoint); }; @@ -481,7 +351,6 @@ export default class HMSTransport { error, task, originalState: this.state, - maxFailedRetries: MAX_TRANSPORT_RETRIES, changeState: false, }); } else { @@ -495,12 +364,15 @@ export default class HMSTransport { this.joinParameters = undefined; HMSLogger.d(TAG, 'leaving in transport'); try { + const usage = this.pluginUsageTracker.getPluginUsage('HMSKrispPlugin'); + if (usage) { + this.eventBus.analytics.publish(AnalyticsEventFactory.getKrispUsage(usage)); + } this.state = TransportState.Leaving; this.publishStatsAnalytics?.stop(); this.subscribeStatsAnalytics?.stop(); this.webrtcInternals?.cleanup(); - await this.publishConnection?.close(); - await this.subscribeConnection?.close(); + this.clearPeerConnections(); if (notifyServer) { try { this.signal.leave(); @@ -539,6 +411,7 @@ export default class HMSTransport { for (const track of tracks) { try { await this.publishTrack(track); + this.connectivityListener?.onMediaPublished(track); } catch (error) { this.eventBus.analytics.publish( AnalyticsEventFactory.publish({ @@ -556,23 +429,128 @@ export default class HMSTransport { } } + setSFUNodeId(id?: string) { + this.signal.setSfuNodeId(id); + if (!this.sfuNodeId) { + this.sfuNodeId = id; + this.publishConnection?.setSfuNodeId(id); + this.subscribeConnection?.setSfuNodeId(id); + } else if (id && this.sfuNodeId !== id) { + this.sfuNodeId = id; + this.handleSFUMigration(); + } + } + + // eslint-disable-next-line complexity + async handleSFUMigration() { + HMSLogger.time('sfu migration'); + this.clearPeerConnections(); + const peers = this.store.getPeerMap(); + this.store.removeRemoteTracks(); + for (const peerId in peers) { + const peer = peers[peerId]; + if (peer.isLocal) { + continue; + } + peer.audioTrack = undefined; + peer.videoTrack = undefined; + peer.auxiliaryTracks = []; + } + + const localPeer = this.store.getLocalPeer(); + if (!localPeer) { + return; + } + this.createPeerConnections(); + this.trackStates.clear(); + await this.negotiateOnFirstPublish(); + const streamMap = new Map(); + if (localPeer.audioTrack) { + const stream = localPeer.audioTrack.stream as HMSLocalStream; + if (!streamMap.get(stream.id)) { + streamMap.set(stream.id, new HMSLocalStream(new MediaStream())); + } + const newTrack = localPeer.audioTrack.clone(streamMap.get(stream.id)!); + this.store.removeTrack(localPeer.audioTrack); + localPeer.audioTrack.cleanup(); + await this.publishTrack(newTrack); + localPeer.audioTrack = newTrack; + } + + if (localPeer.videoTrack) { + const stream = localPeer.videoTrack.stream as HMSLocalStream; + if (!streamMap.get(stream.id)) { + streamMap.set(stream.id, new HMSLocalStream(new MediaStream())); + } + this.store.removeTrack(localPeer.videoTrack); + const newTrack = localPeer.videoTrack.clone(streamMap.get(stream.id)!); + localPeer.videoTrack.cleanup(); + await this.publishTrack(newTrack); + localPeer.videoTrack = newTrack; + } + + const auxTracks = []; + while (localPeer.auxiliaryTracks.length > 0) { + const track = localPeer.auxiliaryTracks.shift(); + if (track) { + const stream = track.stream as HMSLocalStream; + if (!streamMap.get(stream.id)) { + /** + * For screenshare, you need to clone the current stream only, cloning the track will not work otherwise, it will have all + * correct states but bytes sent and all other stats would be 0 + **/ + streamMap.set( + stream.id, + new HMSLocalStream(track.source === 'screen' ? stream.nativeStream.clone() : new MediaStream()), + ); + } + this.store.removeTrack(track); + const newTrack = track.clone(streamMap.get(stream.id)!); + if (newTrack.type === 'video' && newTrack.source === 'screen') { + /** + * Store all the stream so they can be stopped when screenshare stopped. Stopping before is not helping + */ + this.screenStream.add(stream.nativeStream); + this.screenStream.add(newTrack.stream.nativeStream); + newTrack.nativeTrack.addEventListener('ended', this.onScreenshareStop); + } + track.cleanup(); + await this.publishTrack(newTrack); + auxTracks.push(newTrack); + } + } + localPeer.auxiliaryTracks = auxTracks; + streamMap.clear(); + this.listener?.onSFUMigration?.(); + HMSLogger.timeEnd('sfu migration'); + } + /** * TODO: check if track.publishedTrackId be used instead of the hack to match with track with same type and * source. The hack won't work if there are multiple tracks with same source and type. */ - trackUpdate(track: HMSLocalTrack) { + trackUpdate(track: HMSLocalTrack, enabled: boolean) { const currentTrackStates = Array.from(this.trackStates.values()); const originalTrackState = currentTrackStates.find( trackState => track.type === trackState.type && track.source === trackState.source, ); + /** + * on call interruption, we just send disabled track update to biz to send to remote peers WITHOUT sending to the local peer + * in this case, track.enabled would still be true which is why we are using the value from the localVideoEnabled event + * */ if (originalTrackState) { const newTrackState = new TrackState({ ...originalTrackState, - mute: !track.enabled, + mute: !enabled, }); this.trackStates.set(originalTrackState.track_id, newTrackState); HMSLogger.d(TAG, 'Track Update', this.trackStates, track); this.signal.trackUpdate(new Map([[originalTrackState.track_id, newTrackState]])); + const peer = this.store.getLocalPeer(); + // don't send update in case of call interruption + if (peer && enabled === track.enabled) { + this.listener?.onTrackUpdate(enabled ? HMSTrackUpdate.TRACK_UNMUTED : HMSTrackUpdate.TRACK_MUTED, track, peer); + } } } @@ -645,11 +623,32 @@ export default class HMSTransport { stream.removeSender(track); await p; await track.cleanup(); + if (track.source === 'screen' && this.screenStream) { + // stop older screenshare tracks to remove the screenshare banner + this.screenStream.forEach(stream => { + stream.getTracks().forEach(_track => { + _track.stop(); + }); + this.screenStream.delete(stream); + }); + } // remove track from store on unpublish this.store.removeTrack(track); HMSLogger.d(TAG, `✅ unpublishTrack: trackId=${track.trackId}`, this.callbacks); } + private async clearPeerConnections() { + clearTimeout(this.publishDtlsStateTimer); + this.publishDtlsStateTimer = 0; + clearTimeout(this.publishDisconnectTimer); + this.publishDisconnectTimer = 0; + this.lastPublishDtlsState = 'new'; + this.publishConnection?.close(); + this.subscribeConnection?.close(); + this.publishConnection = null; + this.subscribeConnection = null; + } + private waitForLocalRoleAvailability() { if (this.store.hasRoleDetailsArrived()) { return; @@ -680,12 +679,190 @@ export default class HMSTransport { } private createPeerConnections() { + const logConnectionState = ( + role: HMSConnectionRole, + newState: RTCIceConnectionState | RTCPeerConnectionState, + ice = false, + ) => { + const log = ['disconnected', 'failed'].includes(newState) + ? HMSLogger.w.bind(HMSLogger) + : HMSLogger.d.bind(HMSLogger); + + log(TAG, `${HMSConnectionRole[role]} ${ice ? 'ice' : ''} connection state change: ${newState}`); + }; if (this.initConfig) { + const publishConnectionObserver: IPublishConnectionObserver = { + onRenegotiationNeeded: async () => { + await this.performPublishRenegotiation(); + }, + + // eslint-disable-next-line complexity + onDTLSTransportStateChange: (state?: RTCDtlsTransportState) => { + const log = state === 'failed' ? HMSLogger.w.bind(HMSLogger) : HMSLogger.d.bind(HMSLogger); + log(TAG, `Publisher on dtls transport state change: ${state}`); + + if (!state || this.lastPublishDtlsState === state) { + return; + } + + this.lastPublishDtlsState = state; + if (this.publishDtlsStateTimer !== 0) { + clearTimeout(this.publishDtlsStateTimer); + this.publishDtlsStateTimer = 0; + } + + if (state !== 'connecting' && state !== 'failed') { + return; + } + + const timeout = this.initConfig?.config?.dtlsStateTimeouts?.[state]; + if (!timeout || timeout <= 0) { + return; + } + + // if we're in connecting check again after timeout + // hotfix: mitigate https://100ms.atlassian.net/browse/LIVE-1924 + this.publishDtlsStateTimer = window.setTimeout(() => { + const newState = this.publishConnection?.nativeConnection.connectionState; + if (newState && state && newState === state) { + // stuck in either `connecting` or `failed` state for long time + const err = ErrorFactory.WebrtcErrors.ICEFailure( + HMSAction.PUBLISH, + `DTLS transport state ${state} timeout:${timeout}ms`, + true, + ); + this.eventBus.analytics.publish(AnalyticsEventFactory.disconnect(err)); + this.observer.onFailure(err); + } + }, timeout); + }, + + onDTLSTransportError: (error: Error) => { + HMSLogger.e(TAG, `onDTLSTransportError ${error.name} ${error.message}`, error); + this.eventBus.analytics.publish(AnalyticsEventFactory.disconnect(error)); + }, + + onIceConnectionChange: async (newState: RTCIceConnectionState) => { + logConnectionState(HMSConnectionRole.Publish, newState, true); + }, + + onConnectionStateChange: async (newState: RTCPeerConnectionState) => { + logConnectionState(HMSConnectionRole.Publish, newState, false); + if (newState === 'new') { + return; + } + + if (newState === 'connected') { + this.connectivityListener?.onICESuccess(true); + this.publishConnection?.handleSelectedIceCandidatePairs(); + } else if (newState === 'failed') { + await this.handleIceConnectionFailure( + HMSConnectionRole.Publish, + ErrorFactory.WebrtcErrors.ICEFailure( + HMSAction.PUBLISH, + `local candidate - ${this.publishConnection?.selectedCandidatePair?.local?.candidate}; remote candidate - ${this.publishConnection?.selectedCandidatePair?.remote?.candidate}`, + ), + ); + } else { + this.publishDisconnectTimer = window.setTimeout(() => { + if (this.publishConnection?.connectionState !== 'connected') { + this.handleIceConnectionFailure( + HMSConnectionRole.Publish, + ErrorFactory.WebrtcErrors.ICEDisconnected( + HMSAction.PUBLISH, + `local candidate - ${this.publishConnection?.selectedCandidatePair?.local?.candidate}; remote candidate - ${this.publishConnection?.selectedCandidatePair?.remote?.candidate}`, + ), + ); + } + }, ICE_DISCONNECTION_TIMEOUT); + } + }, + + onIceCandidate: candidate => { + this.connectivityListener?.onICECandidate(candidate, true); + }, + + onSelectedCandidatePairChange: candidatePair => { + this.connectivityListener?.onSelectedICECandidatePairChange(candidatePair, true); + }, + }; + + const subscribeConnectionObserver: ISubscribeConnectionObserver = { + onApiChannelMessage: (message: string) => { + this.observer.onNotification(JSON.parse(message)); + }, + + onTrackAdd: (track: HMSTrack) => { + HMSLogger.d(TAG, '[Subscribe] onTrackAdd', `${track}`); + this.observer.onTrackAdd(track); + }, + + onTrackRemove: (track: HMSTrack) => { + HMSLogger.d(TAG, '[Subscribe] onTrackRemove', `${track}`); + this.observer.onTrackRemove(track); + }, + + onIceConnectionChange: async (newState: RTCIceConnectionState) => { + logConnectionState(HMSConnectionRole.Subscribe, newState, true); + + if (newState === 'connected') { + const callback = this.callbacks.get(SUBSCRIBE_ICE_CONNECTION_CALLBACK_ID); + this.callbacks.delete(SUBSCRIBE_ICE_CONNECTION_CALLBACK_ID); + + this.connectivityListener?.onICESuccess(false); + if (callback) { + callback.promise.resolve(true); + } + } + }, + + onConnectionStateChange: async (newState: RTCPeerConnectionState) => { + logConnectionState(HMSConnectionRole.Subscribe, newState, false); + + if (newState === 'failed') { + await this.handleIceConnectionFailure( + HMSConnectionRole.Subscribe, + ErrorFactory.WebrtcErrors.ICEFailure( + HMSAction.SUBSCRIBE, + `local candidate - ${this.subscribeConnection?.selectedCandidatePair?.local?.candidate}; remote candidate - ${this.subscribeConnection?.selectedCandidatePair?.remote?.candidate}`, + ), + ); + } else if (newState === 'disconnected') { + setTimeout(() => { + if (this.subscribeConnection?.connectionState === 'disconnected') { + this.handleIceConnectionFailure( + HMSConnectionRole.Subscribe, + ErrorFactory.WebrtcErrors.ICEDisconnected( + HMSAction.SUBSCRIBE, + `local candidate - ${this.subscribeConnection?.selectedCandidatePair?.local?.candidate}; remote candidate - ${this.subscribeConnection?.selectedCandidatePair?.remote?.candidate}`, + ), + ); + } + }, ICE_DISCONNECTION_TIMEOUT); + } else if (newState === 'connected') { + this.subscribeConnection?.handleSelectedIceCandidatePairs(); + const callback = this.callbacks.get(SUBSCRIBE_ICE_CONNECTION_CALLBACK_ID); + this.callbacks.delete(SUBSCRIBE_ICE_CONNECTION_CALLBACK_ID); + + if (callback) { + callback.promise.resolve(true); + } + } + }, + + onIceCandidate: candidate => { + this.connectivityListener?.onICECandidate(candidate, false); + }, + + onSelectedCandidatePairChange: candidatePair => { + this.connectivityListener?.onSelectedICECandidatePairChange(candidatePair, false); + }, + }; if (!this.publishConnection) { this.publishConnection = new HMSPublishConnection( this.signal, this.initConfig.rtcConfiguration, - this.publishConnectionObserver, + publishConnectionObserver, ); } @@ -694,10 +871,15 @@ export default class HMSTransport { this.signal, this.initConfig.rtcConfiguration, this.isFlagEnabled.bind(this), - this.subscribeConnectionObserver, + subscribeConnectionObserver, ); } } + + this.webrtcInternals?.setPeerConnections({ + publish: this.publishConnection?.nativeConnection, + subscribe: this.subscribeConnection?.nativeConnection, + }); } private async negotiateJoinWithRetry({ @@ -739,7 +921,6 @@ export default class HMSTransport { error: hmsError, task, originalState: TransportState.Joined, - maxFailedRetries: 3, changeState: false, }); } else { @@ -781,6 +962,7 @@ export default class HMSTransport { onDemandTracks, offer, ); + this.setSFUNodeId(answer?.sfu_node_id); await this.publishConnection.setRemoteDescription(answer); for (const candidate of this.publishConnection.candidates) { await this.publishConnection.addIceCandidate(candidate); @@ -803,6 +985,7 @@ export default class HMSTransport { simulcast, onDemandTracks, ); + this.setSFUNodeId(response?.sfu_node_id); return !!response; } @@ -815,22 +998,31 @@ export default class HMSTransport { HMSLogger.e(TAG, 'Publish peer connection not found, cannot negotiate'); return false; } - const offer = await this.publishConnection.createOffer(this.trackStates); - await this.publishConnection.setLocalDescription(offer); - const answer = await this.signal.offer(offer, this.trackStates); - await this.publishConnection.setRemoteDescription(answer); - for (const candidate of this.publishConnection.candidates) { - await this.publishConnection.addIceCandidate(candidate); - } + try { + const offer = await this.publishConnection.createOffer(this.trackStates); + await this.publishConnection.setLocalDescription(offer); + const answer = await this.signal.offer(offer, this.trackStates); + await this.publishConnection.setRemoteDescription(answer); + for (const candidate of this.publishConnection.candidates) { + await this.publishConnection.addIceCandidate(candidate); + } - this.publishConnection.initAfterJoin(); - return !!answer; + this.publishConnection.initAfterJoin(); + return !!answer; + } catch (ex) { + // resolve for now as this might happen during migration + if (ex instanceof HMSException && ex.code === 421) { + return true; + } + throw ex; + } } private async performPublishRenegotiation(constraints?: RTCOfferOptions) { HMSLogger.d(TAG, `⏳ [role=PUBLISH] onRenegotiationNeeded START`, this.trackStates); const callback = this.callbacks.get(RENEGOTIATION_CALLBACK_ID); if (!callback) { + HMSLogger.w(TAG, 'no callback found for renegotiation'); return; } @@ -857,7 +1049,12 @@ export default class HMSTransport { ex = ErrorFactory.GenericErrors.Unknown(HMSAction.PUBLISH, (err as Error).message); } - callback!.promise.reject(ex); + // resolve for now as this might happen during migration + if (ex.code === 421) { + callback.promise.resolve(true); + } else { + callback.promise.reject(ex); + } HMSLogger.d(TAG, `[role=PUBLISH] onRenegotiationNeeded FAILED ❌`); } } @@ -887,12 +1084,11 @@ export default class HMSTransport { error, task: this.retrySubscribeIceFailedTask, originalState: TransportState.Joined, - maxFailedRetries: 1, }); } } - private async internalConnect(token: string, initEndpoint: string, peerId: string) { + private async internalConnect(token: string, initEndpoint: string, peerId: string, iceServers?: HMSICEServer[]) { HMSLogger.d(TAG, 'connect: started ⏰'); const connectRequestedAt = new Date(); try { @@ -906,11 +1102,17 @@ export default class HMSTransport { peerId, userAgent, initEndpoint, + iceServers, }); + this.connectivityListener?.onInitSuccess(this.initConfig.endpoint); const room = this.store.getRoom(); if (room) { room.effectsKey = this.initConfig.config.vb?.effectsKey; room.isEffectsEnabled = this.isFlagEnabled(InitFlags.FLAG_EFFECTS_SDK_ENABLED); + room.disableNoneLayerRequest = this.isFlagEnabled(InitFlags.FLAG_DISABLE_NONE_LAYER_REQUEST); + room.isVBEnabled = this.isFlagEnabled(InitFlags.FLAG_VB_ENABLED); + room.isHipaaEnabled = this.isFlagEnabled(InitFlags.FLAG_HIPAA_ENABLED); + room.isNoiseCancellationEnabled = this.isFlagEnabled(InitFlags.FLAG_NOISE_CANCELLATION); } this.analyticsTimer.end(TimedEvent.INIT); HTTPAnalyticsTransport.setWebsocketEndpoint(this.initConfig.endpoint); @@ -918,6 +1120,7 @@ export default class HMSTransport { this.validateNotDisconnected('post init'); await this.openSignal(token, peerId); this.observer.onConnected(); + this.connectivityListener?.onSignallingSuccess(); this.store.setSimulcastEnabled(this.isFlagEnabled(InitFlags.FLAG_SERVER_SIMULCAST)); HMSLogger.d(TAG, 'Adding Analytics Transport: JsonRpcSignal'); this.analyticsEventsService.setTransport(this.analyticsSignalTransport); @@ -975,15 +1178,6 @@ export default class HMSTransport { HMSLogger.d(TAG, '✅ internal connect: connected to ws endpoint'); } - private async initRtcStatsMonitor() { - this.webrtcInternals?.setPeerConnections({ - publish: this.publishConnection?.nativeConnection, - subscribe: this.subscribeConnection?.nativeConnection, - }); - - this.initStatsAnalytics(); - } - private initStatsAnalytics() { if (this.isFlagEnabled(InitFlags.FLAG_PUBLISH_STATS)) { this.publishStatsAnalytics = new PublishStatsAnalytics( @@ -1093,6 +1287,7 @@ export default class HMSTransport { this.joinParameters!.authToken, this.joinParameters!.endpoint, this.joinParameters!.peerId, + this.joinParameters!.iceServers, ); } @@ -1106,16 +1301,6 @@ export default class HMSTransport { return ok; }; - private handleSubscribeConnectionConnected() { - this.subscribeConnection?.logSelectedIceCandidatePairs(); - const callback = this.callbacks.get(SUBSCRIBE_ICE_CONNECTION_CALLBACK_ID); - this.callbacks.delete(SUBSCRIBE_ICE_CONNECTION_CALLBACK_ID); - - if (callback) { - callback.promise.resolve(true); - } - } - private setTransportStateForConnect() { if (this.state === TransportState.Failed) { this.state = TransportState.Disconnected; diff --git a/packages/hms-video-store/src/transport/models/JoinParameters.ts b/packages/hms-video-store/src/transport/models/JoinParameters.ts index b60829abc7..4e71d97146 100644 --- a/packages/hms-video-store/src/transport/models/JoinParameters.ts +++ b/packages/hms-video-store/src/transport/models/JoinParameters.ts @@ -1,3 +1,5 @@ +import { HMSICEServer } from '../../interfaces'; + export class JoinParameters { constructor( public authToken: string, @@ -6,5 +8,6 @@ export class JoinParameters { public data: string = '', public endpoint: string = 'https://prod-init.100ms.live/init', public autoSubscribeVideo: boolean = false, + public iceServers?: HMSICEServer[], ) {} } diff --git a/packages/hms-video-store/src/utils/constants.ts b/packages/hms-video-store/src/utils/constants.ts index 03c3756d7b..0888adb791 100644 --- a/packages/hms-video-store/src/utils/constants.ts +++ b/packages/hms-video-store/src/utils/constants.ts @@ -3,13 +3,12 @@ export const API_DATA_CHANNEL = 'ion-sfu'; export const ANALYTICS_BUFFER_SIZE = 100; /** - * Maximum number of retries that transport-layer will try + * Maximum time that transport-layer will try * before giving up on the connection and returning a failure * * Refer https://100ms.atlassian.net/browse/HMS-2369 */ -export const MAX_TRANSPORT_RETRIES = 5; -export const MAX_TRANSPORT_RETRY_DELAY = 60; +export const MAX_TRANSPORT_RETRY_TIME = 60_000; export const DEFAULT_SIGNAL_PING_TIMEOUT = 12_000; export const DEFAULT_SIGNAL_PING_INTERVAL = 3_000; @@ -34,10 +33,14 @@ export const PUBLISH_STATS_PUSH_INTERVAL = 300; export const SUBSCRIBE_STATS_SAMPLE_WINDOW = 10; export const SUBSCRIBE_STATS_PUSH_INTERVAL = 60; +export const MAX_SAFE_INTEGER = Math.pow(2, 31) - 1; + export const HMSEvents = { DEVICE_CHANGE: 'device-change', LOCAL_AUDIO_ENABLED: 'local-audio-enabled', LOCAL_VIDEO_ENABLED: 'local-video-enabled', + LOCAL_VIDEO_UNMUTED_NATIVELY: 'local-video-unmuted-natively', + LOCAL_AUDIO_UNMUTED_NATIVELY: 'local-audio-unmuted-natively', STATS_UPDATE: 'stats-update', // emitted by HMSWebrtcInternals RTC_STATS_UPDATE: 'rtc-stats-update', // emitted by RTCStatsMonitor TRACK_DEGRADED: 'track-degraded', @@ -57,10 +60,14 @@ export const HMSEvents = { export const PROTOCOL_VERSION = '2.5'; -export const PROTOCOL_SPEC = '20231201'; +export const PROTOCOL_SPEC = '20240720'; export const HAND_RAISE_GROUP_NAME = '_handraise'; export const DEFAULT_PLAYLIST_VIDEO_BITRATE = 1000; export const DEFAULT_PLAYLIST_AUDIO_BITRATE = 64; + +export const WHITEBOARD_ORIGIN = 'https://whiteboard.100ms.live'; + +export const WHITEBOARD_QA_ORIGIN = 'https://whiteboard-qa.100ms.live'; diff --git a/packages/hms-video-store/src/utils/ice-server-config.ts b/packages/hms-video-store/src/utils/ice-server-config.ts new file mode 100644 index 0000000000..4af82722f1 --- /dev/null +++ b/packages/hms-video-store/src/utils/ice-server-config.ts @@ -0,0 +1,11 @@ +import { HMSICEServer } from '../interfaces'; + +export const transformIceServerConfig = (defaultConfig?: RTCIceServer[], iceServers?: HMSICEServer[]) => { + if (!iceServers || iceServers.length === 0) { + return defaultConfig; + } + const transformedIceServers = iceServers.map(server => { + return { urls: server.urls, credentialType: 'password', credential: server.password, username: server.userName }; + }); + return transformedIceServers; +}; diff --git a/packages/hms-video-store/src/utils/media.ts b/packages/hms-video-store/src/utils/media.ts index 9e669e61aa..05d60e443f 100644 --- a/packages/hms-video-store/src/utils/media.ts +++ b/packages/hms-video-store/src/utils/media.ts @@ -1,12 +1,13 @@ import HMSLogger from './logger'; -import { BuildGetMediaError, HMSGetMediaActions } from '../error/utils'; +import { BuildGetMediaError } from '../error/utils'; +import { HMSTrackExceptionTrackType } from '../media/tracks/HMSTrackExceptionTrackType'; export async function getLocalStream(constraints: MediaStreamConstraints): Promise { try { const stream = await navigator.mediaDevices.getUserMedia(constraints); return stream; } catch (err) { - throw BuildGetMediaError(err as Error, HMSGetMediaActions.AV); + throw BuildGetMediaError(err as Error, HMSTrackExceptionTrackType.AUDIO_VIDEO); } } @@ -16,7 +17,7 @@ export async function getLocalScreen(constraints: MediaStreamConstraints['video' const stream = await navigator.mediaDevices.getDisplayMedia({ video: constraints, audio: false }); return stream; } catch (err) { - throw BuildGetMediaError(err as Error, HMSGetMediaActions.SCREEN); + throw BuildGetMediaError(err as Error, HMSTrackExceptionTrackType.SCREEN); } } @@ -37,7 +38,7 @@ export async function getLocalDevices(): Promise { devices.forEach(device => deviceGroups[device.kind].push(device)); return deviceGroups; } catch (err) { - throw BuildGetMediaError(err as Error, HMSGetMediaActions.AV); + throw BuildGetMediaError(err as Error, HMSTrackExceptionTrackType.AUDIO_VIDEO); } } @@ -63,3 +64,28 @@ export const HMSAudioContextHandler: HMSAudioContext = { } }, }; + +export enum HMSAudioDeviceCategory { + SPEAKERPHONE = 'SPEAKERPHONE', + WIRED = 'WIRED', + BLUETOOTH = 'BLUETOOTH', + EARPIECE = 'EARPIECE', +} + +export const getAudioDeviceCategory = (deviceLabel?: string) => { + if (!deviceLabel) { + HMSLogger.w('[DeviceManager]:', 'No device label provided'); + return HMSAudioDeviceCategory.SPEAKERPHONE; + } + const label = deviceLabel.toLowerCase(); + if (label.includes('speakerphone')) { + return HMSAudioDeviceCategory.SPEAKERPHONE; + } else if (label.includes('wired')) { + return HMSAudioDeviceCategory.WIRED; + } else if (/airpods|buds|wireless|bluetooth/gi.test(label)) { + return HMSAudioDeviceCategory.BLUETOOTH; + } else if (label.includes('earpiece')) { + return HMSAudioDeviceCategory.EARPIECE; + } + return HMSAudioDeviceCategory.SPEAKERPHONE; +}; diff --git a/packages/hms-video-store/src/utils/support.ts b/packages/hms-video-store/src/utils/support.ts index a0badd0de4..ea53847475 100644 --- a/packages/hms-video-store/src/utils/support.ts +++ b/packages/hms-video-store/src/utils/support.ts @@ -30,3 +30,5 @@ export const isPageHidden = () => typeof document !== 'undefined' && document.hi export const isIOS = () => parsedUserAgent.getOS().name?.toLowerCase() === 'ios'; export const isFirefox = parsedUserAgent.getBrowser()?.name?.toLowerCase().includes('firefox'); +// safari for mac and mobile safari for iOS +export const isSafari = parsedUserAgent.getBrowser()?.name?.toLowerCase().includes('safari'); diff --git a/packages/hms-video-store/src/utils/timer-utils.ts b/packages/hms-video-store/src/utils/timer-utils.ts index 717224ee89..c7cda0717b 100644 --- a/packages/hms-video-store/src/utils/timer-utils.ts +++ b/packages/hms-video-store/src/utils/timer-utils.ts @@ -1,4 +1,4 @@ -export const worker = `(function metronomeWorkerSetup() { +export const worker = `(function workerSetup() { function ticker() { self.postMessage('tick'); } @@ -47,6 +47,27 @@ export function workerSleep(ms: number): Promise { }); } +export function reusableWorker() { + if (typeof Worker === 'undefined') { + return { + sleep: (ms: number) => sleep(ms), + }; + } + const WorkerThread = new Worker(URL.createObjectURL(new Blob([worker], { type: 'application/javascript' }))); + return { + sleep: (ms: number) => { + WorkerThread.postMessage(['start', ms]); + return new Promise(resolve => { + WorkerThread.onmessage = event => { + if (event.data === 'tick') { + resolve(); + } + }; + }); + }, + }; +} + /** * Debounce Fn - Function to limit the number of executions of the passed in * function in a given time duration diff --git a/packages/hms-video-store/src/utils/track-audio-level-monitor.ts b/packages/hms-video-store/src/utils/track-audio-level-monitor.ts index d74b880af9..c191424a87 100644 --- a/packages/hms-video-store/src/utils/track-audio-level-monitor.ts +++ b/packages/hms-video-store/src/utils/track-audio-level-monitor.ts @@ -16,10 +16,12 @@ export interface ITrackAudioLevelUpdate { audioLevel: number; } +// Audio level algorithm referenced from official MDN example - https://github.com/mdn/dom-examples/tree/main/media/web-dictaphone export class TrackAudioLevelMonitor { private readonly TAG = '[TrackAudioLevelMonitor]'; private audioLevel = 0; private analyserNode?: AnalyserNode; + private dataArray?: Uint8Array; private isMonitored = false; /** Frequency of polling audio level from track */ private interval = 100; @@ -35,6 +37,8 @@ export class TrackAudioLevelMonitor { try { const stream = new MediaStream([this.track.nativeTrack]); this.analyserNode = this.createAnalyserNodeForStream(stream); + const bufferLength = this.analyserNode.frequencyBinCount; + this.dataArray = new Uint8Array(bufferLength); } catch (ex) { HMSLogger.w(this.TAG, 'Unable to initialize AudioContext', ex); } @@ -113,16 +117,15 @@ export class TrackAudioLevelMonitor { } private calculateAudioLevel() { - if (!this.analyserNode) { + if (!this.analyserNode || !this.dataArray) { HMSLogger.d(this.TAG, 'AudioContext not initialized'); return; } - const data = new Uint8Array(this.analyserNode.fftSize); - this.analyserNode.getByteTimeDomainData(data); + this.analyserNode.getByteTimeDomainData(this.dataArray); const lowest = 0.009; let max = lowest; - for (const frequency of data) { + for (const frequency of this.dataArray) { max = Math.max(max, (frequency - 128) / 128); } const normalized = (Math.log(lowest) - Math.log(max)) / Math.log(lowest); @@ -130,23 +133,23 @@ export class TrackAudioLevelMonitor { return percent; } - private isSilentThisInstant() { - if (!this.analyserNode) { + isSilentThisInstant() { + if (!this.analyserNode || !this.dataArray) { HMSLogger.d(this.TAG, 'AudioContext not initialized'); return; } - const data = new Uint8Array(this.analyserNode.fftSize); - this.analyserNode.getByteTimeDomainData(data); + this.analyserNode.getByteTimeDomainData(this.dataArray); // For absolute silence(in case of mic/software failures), all frequencies are 128 or 0. - return !data.some(frequency => frequency !== 128 && frequency !== 0); + return this.dataArray.every(frequency => frequency === 128 || frequency === 0); } private createAnalyserNodeForStream(stream: MediaStream): AnalyserNode { const audioContext = HMSAudioContextHandler.getAudioContext(); - const analyser = audioContext.createAnalyser(); const source = audioContext.createMediaStreamSource(stream); + const analyser = audioContext.createAnalyser(); + analyser.fftSize = 2048; source.connect(analyser); return analyser; } diff --git a/packages/hms-video-store/src/utils/track.ts b/packages/hms-video-store/src/utils/track.ts index fea545dd08..6af65ce30f 100644 --- a/packages/hms-video-store/src/utils/track.ts +++ b/packages/hms-video-store/src/utils/track.ts @@ -1,5 +1,6 @@ -import { BuildGetMediaError, HMSGetMediaActions } from '../error/utils'; +import { BuildGetMediaError } from '../error/utils'; import { HMSAudioTrackSettings, HMSVideoTrackSettings } from '../media/settings'; +import { HMSTrackExceptionTrackType } from '../media/tracks/HMSTrackExceptionTrackType'; export async function getAudioTrack(settings: HMSAudioTrackSettings): Promise { try { @@ -8,7 +9,7 @@ export async function getAudioTrack(settings: HMSAudioTrackSettings): Promise { throw error; } }; + +export const validatePublishParams = (store: Store) => { + const publishParams = store.getPublishParams(); + if (!publishParams) { + throw ErrorFactory.GenericErrors.NotConnected( + HMSAction.VALIDATION, + 'call addTrack after preview or join is successful', + ); + } +}; diff --git a/packages/hms-video-store/src/utils/whiteboard.ts b/packages/hms-video-store/src/utils/whiteboard.ts new file mode 100644 index 0000000000..22ed044271 --- /dev/null +++ b/packages/hms-video-store/src/utils/whiteboard.ts @@ -0,0 +1,12 @@ +import { WHITEBOARD_ORIGIN, WHITEBOARD_QA_ORIGIN } from './constants'; +import { ENV } from './support'; + +export const constructWhiteboardURL = (token: string, addr: string, env?: ENV) => { + const origin = env === ENV.QA ? WHITEBOARD_QA_ORIGIN : WHITEBOARD_ORIGIN; + + const url = new URL(origin); + url.searchParams.set('endpoint', `https://${addr}`); + url.searchParams.set('token', token); + + return url.toString(); +}; diff --git a/packages/hms-virtual-background/.gitignore b/packages/hms-virtual-background/.gitignore deleted file mode 100755 index 4c9d7c35a4..0000000000 --- a/packages/hms-virtual-background/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -*.log -.DS_Store -node_modules -dist diff --git a/packages/hms-virtual-background/README.md b/packages/hms-virtual-background/README.md index ac8a38d7c4..75ec13a8ec 100755 --- a/packages/hms-virtual-background/README.md +++ b/packages/hms-virtual-background/README.md @@ -1,13 +1,243 @@ -## installation +# Virtual Background with Effects SDK -**with npm:** +[![Lint, Test and Build](https://github.com/100mslive/web-sdks/actions/workflows/lint-test-build.yml/badge.svg)](https://github.com/100mslive/web-sdks/actions/workflows/lint-test-build.yml) +[![Bundle Size](https://badgen.net/bundlephobia/minzip/@100mslive/hms-virtual-background)](https://bundlephobia.com/result?p=@100mslive/hms-virtual-background) +[![License](https://img.shields.io/npm/l/@100mslive/hms-virtual-background)](https://www.100ms.live/) +![Tree shaking](https://badgen.net/bundlephobia/tree-shaking/@100mslive/hms-virtual-background) -```npm i --save @100mslive/hms-virtual-background``` +Virtual background plugin helps in customising one’s background. The customising options are blurring the background or replacing it with a static image. This guide provides an overview of usage of the virtual background plugin of 100ms. -**with yarn** +## Pre-requisites -```yarn add @100mslive/hms-virtual-background``` +Get the 100ms VirtualBackground Package** (Supported since version 1.11.28) -## Usage +```bash section=GetHMSVirtualBackgroundPackage sectionIndex=1 +npm install --save @100mslive/hms-virtual-background@latest +``` -Refer our [docs](https://www.100ms.live/docs/javascript/v2/plugins/virtual-background#start-and-stop-virtual-background) for usage. \ No newline at end of file +## Features + +The following features are currently supported under the `HMSEffectsPlugin` class: + +```js +/** + * Sets the blur intensity. + * @param {number} blur - The blur intensity, ranging from 0 to 1. + * @returns {void} + */ +setBlur(blur) {} + +/** + * Sets the virtual background using the provided media URL. + * @param {HMSEffectsBackground} url - The media URL to set as the virtual background. + * Alternatively, the background image can be downloaded beforehand and passed to setBackground as objectURL + * @returns {void} + */ +setBackground(url) {} + +/** + * Retrieves the name of the plugin. + * @returns {string} The name of the plugin, 'HMSEffects'. + */ +getName() {} + +/** + * Retrieves the currently enabled background type or media URL. + * @returns {string|MediaStream|MediaStreamTrack|HTMLVideoElement} The background type or media URL. + */ +getBackground() {} + +/** + * Sets the preset quality of the virtual background. + * Options: "balanced" | "quality" + * The 'quality' preset has a higher CPU usage than the 'balanced' preset which is the default + * @param {string} preset - The preset quality to set. + * @returns {Promise} + */ +setPreset(preset) {} + +/** + * Retrieves the active preset quality of the virtual background. + * @returns {string} The active preset quality. + */ +getPreset() {} + +/** + * Clears all applied filters. + * @returns {void} + */ +removeEffects() {} + +/** + * Stops the plugin. + * @returns {void} + */ +stop() {} + +``` + +Callbacks supported by the plugin: + +On initialization, after the required resources are downloaded by the plugin: + +``` +const effectsPlugin = new HMSEffectsPlugin(, () => console.log("Plugin initialised")); + +``` + +On resolution change (on device rotation or when the video aspect ratio changes): + +``` +effectsPlugin.onResolutionChange = (width: number, height: number) => { + console.log(`Resolution changed to ${width}x${height}`) +} +``` + +## Instantiate Virtual Background + +The SDK key for effects is needed to instantiate the `HMSEffectsPlugin` class: + +```jsx + const effectsKey = useHMSStore(selectEffectsKey); +``` + +It is recommended to initialise the object in a separate file to prevent multiple initialisations on re-renders and keep the UI level code independent of internal calls by the SDK. + + +```js +import { HMSEffectsPlugin, HMSVirtualBackgroundTypes } from '@100mslive/hms-virtual-background'; + +export class VBPlugin { + private effectsPlugin?: HMSEffectsPlugin | undefined; + + initialisePlugin = (effectsSDKKey?: string) => { + if (this.getVBObject()) { + return; + } + if (effectsSDKKey) { + this.effectsPlugin = new HMSEffectsPlugin(effectsSDKKey); + } + }; + + getBackground = () => { + return this.effectsPlugin?.getBackground(); + }; + + getBlurAmount = () => { + return this.effectsPlugin?.getBlurAmount(); + }; + + getVBObject = () => { + return this.effectsPlugin; + }; + + getName = () => { + return this.effectsPlugin?.getName(); + }; + + setBlur = async (blurPower: number) => { + this.effectsPlugin?.setBlur(blurPower); + }; + + setBackground = async (mediaURL: string) => { + this.effectsPlugin?.setBackground(mediaURL); + }; + + setPreset = (preset: string) => { + this.effectsPlugin.setPreset(preset); + }; + + getPreset = () => { + return this.effectsPlugin?.getPreset() || ''; + }; + + removeEffects = async () => { + this.effectsPlugin?.removeEffects(); + }; + + reset = () => { + this.effectsPlugin = undefined; + }; +} + +export const VBHandler = new VBPlugin(); +``` + +## Building the UI + +The following snippet illustrates how to add the plugin to the video and manage the UI state to preserve configuration: + +```jsx +import { + selectEffectsKey, + selectIsLocalVideoPluginPresent + selectLocalVideoTrackID, + useHMSStore, +} from '@100mslive/react-sdk'; +import { + HMSEffectsPlugin, + HMSVirtualBackgroundTypes +} from '@100mslive/hms-virtual-background'; +import { VBHandler } from './VBHandler'; + +export const VirtualBackgroundPicker = () => { + const hmsActions = useHMSActions(); + // Get the effects SDK key here + const effectsKey = useHMSStore(selectEffectsKey); + const trackId = useHMSStore(selectLocalVideoTrackID); + const isPluginAdded = useHMSStore(selectIsLocalVideoPluginPresent(VBHandler?.getName() || '')); + + // State can be used to show active selection + const [background, setBackground] = useState( + VBHandler.getBackground() as string | HMSVirtualBackgroundTypes, + ); + + useEffect(() => { + if (!track?.id) { + return; + } + if (!isPluginAdded) { + let vbObject = VBHandler.getVBObject(); + if (!vbObject) { + VBHandler.initialisePlugin(effectsKey); + vbObject = VBHandler.getVBObject(); + if (effectsKey) { + hmsActions.addPluginsToVideoStream([vbObject as HMSEffectsPlugin]); + } + } + } + }, [hmsActions, isPluginAdded, effectsKey, track?.id]); + + // UI code for media picker can go here +} + +``` + +This handles initialisation and adding the plugin to the video stream. The plugin takes a few seconds on first load during initialisation. Subsequent filter and effect selections should take less than a second to reflect. + +The methods can be called via the `VBHandler` object: +```jsx +const setBackground = async(mediaURL : string) => { + await VBHandler?.setBackground(mediaURL); + // The selection can be highlighted using the activeBackground state + setActiveBackground(mediaURL); +} + +const setBlur = async(blurAmount: number) => { + await VBHandler?.setBlur(blurAmount); + setActiveBackground(HMSVirtualBackgroundTypes.BLUR); +} + +const removeEffects = async() => { + await VBHandler.removeEffects(); + setActiveBackground(HMSVirtualBackgroundTypes.NONE); +} +``` + + +The full implementation can be viewed in the [roomkit-react package](https://github.com/100mslive/web-sdks/blob/main/packages/roomkit-react/src/Prebuilt/components/VirtualBackground/VBPicker.tsx). + + +## Supported Browsers + +Effects virtual background is supported on Safari, Firefox and Chromium based browsers.
diff --git a/packages/hms-virtual-background/package.json b/packages/hms-virtual-background/package.json index d79ec979b5..0df815ac53 100755 --- a/packages/hms-virtual-background/package.json +++ b/packages/hms-virtual-background/package.json @@ -1,11 +1,41 @@ { - "version": "1.12.6", + "version": "1.13.25", "license": "MIT", "name": "@100mslive/hms-virtual-background", "author": "100ms", - "module": "dist/index.js", - "main": "dist/index.cjs.js", + "module": "dist/esm/index.js", + "main": "dist/cjs/index.js", "typings": "dist/index.d.ts", + "typesVersions": { + "*": { + "hmsvbplugin": [ + "./dist/HMSVBPlugin.d.ts" + ], + "hmseffectsplugin": [ + "./dist/HMSEffectsPlugin.d.ts" + ] + } + }, + "exports": { + ".": { + "import": "./dist/esm/index.js", + "require": "./dist/cjs/index.js", + "default": "./dist/esm/index.js", + "types": "./dist/index.d.ts" + }, + "./hmsvbplugin": { + "import": "./dist/esm/HMSVBPlugin.js", + "require": "./dist/cjs/HMSVBPlugin.js", + "default": "./dist/esm/HMSVBPlugin.js", + "types": "./dist/HMSVBPlugin.d.ts" + }, + "./hmseffectsplugin": { + "import": "./dist/esm/HMSEffectsPlugin.js", + "default": "./dist/esm/HMSEffectsPlugin.js", + "require": "./dist/cjs/HMSEffectsPlugin.js", + "types": "./dist/HMSEffectsPlugin.d.ts" + } + }, "repository": { "type": "git", "url": "https://github.com/100mslive/web-sdks.git", @@ -32,10 +62,10 @@ "format": "prettier --write src/**/*.ts" }, "peerDependencies": { - "@100mslive/hms-video-store": "0.11.6" + "@100mslive/hms-video-store": "0.12.25" }, "devDependencies": { - "@100mslive/hms-video-store": "0.11.6" + "@100mslive/hms-video-store": "0.12.25" }, "dependencies": { "@mediapipe/selfie_segmentation": "^0.1.1632777926", @@ -45,7 +75,7 @@ "@tensorflow/tfjs-core": "^3.19.0", "@webassemblyjs/helper-wasm-bytecode": "1.11.1", "@webassemblyjs/wasm-gen": "1.11.1", - "effects-sdk": "2.6.8", + "effects-sdk": "3.2.3", "gifuct-js": "^2.1.2", "wasm-check": "^2.0.2" }, diff --git a/packages/hms-virtual-background/src/HMSEffectsPlugin.ts b/packages/hms-virtual-background/src/HMSEffectsPlugin.ts index b8a3ed5f5b..43105e33d1 100644 --- a/packages/hms-virtual-background/src/HMSEffectsPlugin.ts +++ b/packages/hms-virtual-background/src/HMSEffectsPlugin.ts @@ -7,14 +7,20 @@ export type HMSEffectsBackground = string | MediaStream | MediaStreamTrack | HTM export class HMSEffectsPlugin implements HMSMediaStreamPlugin { private effects: tsvb; - // Ranges from 0 to 1 + // Ranges from 0 to 1, inclusive private blurAmount = 0; private background: HMSEffectsBackground = HMSVirtualBackgroundTypes.NONE; private backgroundType = HMSVirtualBackgroundTypes.NONE; - private preset = 'balanced'; + private preset: 'balanced' | 'quality' = 'balanced'; + private initialised = false; + private intervalId: NodeJS.Timer | null = null; + private onInit; + private onResolutionChangeCallback?: (width: number, height: number) => void; + private canvas: HTMLCanvasElement; - constructor(effectsSDKKey: string) { + constructor(effectsSDKKey: string, onInit?: () => void) { this.effects = new tsvb(effectsSDKKey); + this.onInit = onInit; this.effects.config({ sdk_url: EFFECTS_SDK_ASSETS, models: { @@ -25,36 +31,88 @@ export class HMSEffectsPlugin implements HMSMediaStreamPlugin { wasmPaths: { 'ort-wasm.wasm': `${EFFECTS_SDK_ASSETS}ort-wasm.wasm`, 'ort-wasm-simd.wasm': `${EFFECTS_SDK_ASSETS}ort-wasm-simd.wasm`, - 'ort-wasm-threaded.wasm': `${EFFECTS_SDK_ASSETS}ort-wasm-threaded.wasm`, - 'ort-wasm-simd-threaded.wasm': `${EFFECTS_SDK_ASSETS}ort-wasm-simd-threaded.wasm`, }, }); + this.canvas = document.createElement('canvas'); + this.effects.onError(err => { + // currently logging info type messages as well + if (!err.type || err.type === 'error') { + console.error('[HMSEffectsPlugin]', err); + } + }); + this.effects.cache(); + this.effects.onReady = () => { + if (this.effects) { + this.initialised = true; + this.onInit?.(); + this.effects.run(); + this.effects.setBackgroundFitMode('fill'); + this.effects.setSegmentationPreset(this.preset); + this.applyEffect(); + } + }; } getName(): string { return 'HMSEffects'; } + private executeAfterInit(callback: () => void) { + if (this.initialised) { + callback(); + } + + if (this.intervalId !== null) { + clearInterval(this.intervalId); + } + this.intervalId = setInterval(() => { + if (this.initialised) { + clearInterval(this.intervalId!); + callback(); + } + }, 100); + } + removeBlur() { this.blurAmount = 0; - this.effects.clearBlur(); + this.executeAfterInit(() => { + this.effects.clearBlur(); + }); } removeBackground() { this.background = ''; - this.effects.clearBackground(); + this.executeAfterInit(() => { + this.effects.clearBackground(); + }); } + /** + * @param blur ranges between 0 and 1 + */ setBlur(blur: number) { - this.removeBackground(); this.blurAmount = blur; this.backgroundType = HMSVirtualBackgroundTypes.BLUR; - this.effects.setBlur(blur); + this.removeBackground(); + this.executeAfterInit(() => { + this.effects.setBlur(this.blurAmount); + }); } - async setPreset(preset: string) { + /** + * @param preset can be 'quality' or 'balanced'. The 'quality' preset has better quality but higher CPU usage than 'balanced' + */ + async setPreset(preset: 'quality' | 'balanced') { this.preset = preset; - await this.effects.setSegmentationPreset(this.preset); + return new Promise((resolve, reject) => { + this.executeAfterInit(() => { + this.effects.setSegmentationPreset(this.preset).then(resolve).catch(reject); + }); + }); + } + + onResolutionChange(callback: (width: number, height: number) => void) { + this.onResolutionChangeCallback = callback; } getPreset() { @@ -69,9 +127,11 @@ export class HMSEffectsPlugin implements HMSMediaStreamPlugin { setBackground(url: HMSEffectsBackground) { this.background = url; - this.removeBlur(); this.backgroundType = HMSVirtualBackgroundTypes.IMAGE; - this.effects.setBackground(url); + this.removeBlur(); + this.executeAfterInit(() => { + this.effects.setBackground(this.background); + }); } getBlurAmount() { @@ -82,27 +142,38 @@ export class HMSEffectsPlugin implements HMSMediaStreamPlugin { return this.background || this.backgroundType; } + private updateCanvas(stream: MediaStream) { + const { height, width } = stream.getVideoTracks()[0].getSettings(); + this.canvas.width = width!; + this.canvas.height = height!; + this.effects.useStream(stream); + this.effects.toCanvas(this.canvas); + } + apply(stream: MediaStream): MediaStream { - this.effects.onReady = () => { - if (this.effects) { - this.effects.run(); - this.effects.setBackgroundFitMode('fill'); - this.effects.setSegmentationPreset(this.preset); - if (this.blurAmount) { - this.setBlur(this.blurAmount); - } else if (this.background) { - this.setBackground(this.background); - } - } - }; this.effects.clear(); - this.effects.useStream(stream); - // getStream potentially returns null - return this.effects.getStream() || stream; + this.applyEffect(); + this.effects.onChangeInputResolution(() => { + this.updateCanvas(stream); + const { height, width } = stream.getVideoTracks()[0].getSettings(); + this.onResolutionChangeCallback?.(width!, height!); + }); + this.updateCanvas(stream); + return this.canvas.captureStream(30) || stream; } stop() { this.removeEffects(); - this.effects.stop(); + this.executeAfterInit(() => { + this.effects.stop(); + }); + } + + private applyEffect() { + if (this.blurAmount) { + this.setBlur(this.blurAmount); + } else if (this.background) { + this.setBackground(this.background); + } } } diff --git a/packages/hms-virtual-background/src/HMSVBPlugin.ts b/packages/hms-virtual-background/src/HMSVBPlugin.ts index b94ce59494..7e89dcee9c 100644 --- a/packages/hms-virtual-background/src/HMSVBPlugin.ts +++ b/packages/hms-virtual-background/src/HMSVBPlugin.ts @@ -42,9 +42,13 @@ export class HMSVBPlugin implements HMSVideoPlugin { return this.checkSupport().isSupported; } + isBlurSupported(): boolean { + return 'filter' in CanvasRenderingContext2D.prototype; + } + checkSupport(): HMSPluginSupportResult { const browserResult = {} as HMSPluginSupportResult; - if (['Chrome', 'Firefox', 'Edg', 'Edge'].some(value => navigator.userAgent.indexOf(value) !== -1)) { + if (['Chrome', 'Firefox', 'Edg', 'Edge', 'Safari'].some(value => navigator.userAgent.indexOf(value) !== -1)) { browserResult.isSupported = true; } else { browserResult.isSupported = false; @@ -296,3 +300,5 @@ export class HMSVBPlugin implements HMSVideoPlugin { this.renderBackground(results, this.tempGifCanvas); } } + +export { HMSVirtualBackgroundTypes }; diff --git a/packages/hms-virtual-background/src/constants.ts b/packages/hms-virtual-background/src/constants.ts index b6070950de..9c8b8a75da 100644 --- a/packages/hms-virtual-background/src/constants.ts +++ b/packages/hms-virtual-background/src/constants.ts @@ -1 +1 @@ -export const EFFECTS_SDK_ASSETS = 'https://assets.100ms.live/effectsdk/'; +export const EFFECTS_SDK_ASSETS = 'https://assets.100ms.live/effectsdk/3.2.3/'; diff --git a/packages/hms-virtual-background/src/index.ts b/packages/hms-virtual-background/src/index.ts index b2c0fb424d..5bcf425e45 100644 --- a/packages/hms-virtual-background/src/index.ts +++ b/packages/hms-virtual-background/src/index.ts @@ -1,5 +1,5 @@ export * from './HMSVirtualBackgroundPlugin'; -export * from './HMSVBPlugin'; +export { HMSVBPlugin } from './HMSVBPlugin'; export { HMSVirtualBackgroundTypes } from './interfaces'; export type { HMSVirtualBackground } from './interfaces'; export { HMSEffectsPlugin } from './HMSEffectsPlugin'; diff --git a/packages/hms-whiteboard/.eslintrc b/packages/hms-whiteboard/.eslintrc new file mode 100644 index 0000000000..3bac8a72fa --- /dev/null +++ b/packages/hms-whiteboard/.eslintrc @@ -0,0 +1,87 @@ +{ + "extends": [ + "eslint:recommended", + "plugin:@typescript-eslint/recommended", + "plugin:import/recommended", + "plugin:import/typescript", + "react-app", + "react-app/jest" + ], + "plugins": ["prettier", "simple-import-sort"], + "rules": { + "@typescript-eslint/ban-ts-comment": 0, + "@typescript-eslint/no-unused-vars": "error", + "no-nested-ternary": 1, + "no-unused-vars": [ + "error", + { + "vars": "all", + "args": "none", + "ignoreRestSiblings": false + } + ], + "react/react-in-jsx-scope": "error", + "react/jsx-curly-brace-presence": [ + 2, + { + "props": "never" + } + ], + "react/self-closing-comp": [ + "error", + { + "component": true, + "html": false + } + ], + "prettier/prettier": [ + "error", + { + "endOfLine": "auto" // This is need to handle different end-of-line in windows/mac + } + ], + "import/first": "error", + "import/namespace": [2, { "allowComputed": true }], + "import/no-duplicates": "error", + "simple-import-sort/imports": [ + "error", + { + "groups": [ + [ + // Packages `react` related packages come first. + "^react", + "^@?\\w", + // Internal packages. + "^@100mslive/react-sdk", + "^@100mslive/react-icons", + // Side effect imports. + "^\\u0000", + + "(components)", + // Other relative imports. Put same-folder imports and `.`. + "^\\./(?=.*/)(?!/?$)", + "^\\.(?!/?$)", + "^\\./?$", + "(plugins)", + "(components)?(.*)(/use.*)", + ".*(hooks)", + "(common)", + "(services)", + "(utils)", + "(constants)", + // Style imports. + "^.+\\.?(css)$" + ] + ] + } + ] + }, + "settings": { + "import/resolver": { + "node": { + "extensions": [".js", ".jsx", ".ts", ".tsx"] + } + } + }, + "ignorePatterns": ["src/*.css", "src/images/*", "src/grpc/*", "dist/**"] +} diff --git a/packages/hms-whiteboard/README.md b/packages/hms-whiteboard/README.md new file mode 100644 index 0000000000..23137323d7 --- /dev/null +++ b/packages/hms-whiteboard/README.md @@ -0,0 +1,125 @@ +# 100ms Whiteboard + +[![Lint, Test and Build](https://github.com/100mslive/web-sdks/actions/workflows/lint-test-build.yml/badge.svg)](https://github.com/100mslive/web-sdks/actions/workflows/lint-test-build.yml) +[![Bundle Size](https://badgen.net/bundlephobia/minzip/@100mslive/hms-whiteboard)](https://bundlephobia.com/result?p=@100mslive/hms-whiteboard) +[![License](https://img.shields.io/npm/l/@100mslive/hms-whiteboard)](https://www.100ms.live/) +![Tree shaking](https://badgen.net/bundlephobia/tree-shaking/@100mslive/hms-whiteboard) + +The 100ms SDK provides robust APIs for integrating whiteboard collaboration into your conferencing sessions. Participants can engage in real-time by drawing, writing, and collaborating on a shared digital whiteboard. This documentation outlines how to implement the start and stop functionality for a whiteboard and display it within an iframe or embed it as a React component. + +## Requirements + +- React 18 or higher +- Webpack 5 or higher if you're using it to bundle your app +- User roles must be configured to enable whiteboard functionality via the 100ms dashboard for starting or viewing the whiteboard. [Refer here](https://www.100ms.live/docs/get-started/v2/get-started/features/whiteboard#enabling-and-configuring-the-whiteboard). +- If you're on React and are not using the `@100mslive/roomkit-react` package, install the `@100mslive/hms-whiteboard` package. + +```bash +yarn add @100mslive/hms-whiteboard +``` + +## Opening and Closing the Whiteboard + +JavaScript users can use the `selectPermissions` selector which fetches the whiteboard specific permissions array from the local peer's role permissions. + +React users can check for the `toggle` function returned by the utility hook `useWhiteboard`. + +```js +// Vanilla JavaScript Example +import { selectPermissions, selectWhiteboard } from '@100mslive/hms-video-store'; + +const permissions = hmsStore.getState(selectPermissions)?.whiteboard; // Array<'read' | 'write' | 'admin'> +const isAdmin = !!permissions?.includes('admin'); +const whiteboard = hmsStore.getState(selectWhiteboard); +const isOwner = whiteboard?.owner === localPeerUserId; + +const toggle = async () => { + if (!isAdmin) { + return; + } + + if (whiteboard?.open) { + isOwner && (await actions.interactivityCenter.whiteboard.close()); + } else { + await actions.interactivityCenter.whiteboard.open(); + } +}; + +// usage +const toggleButton = document.getElementById('toggle-whiteboard'); +// non-admin users cannot toggle the whiteboard +toggleButton.disabled = !isAdmin; +toggleButton.onclick = toggle; +``` + +```jsx +// React Example +import React from 'react'; +import { useWhiteboard } from '@100mslive/react-sdk'; + +export const WhiteboardToggle = () => { + const { toggle, open, isOwner } = useWhiteboard(); + + // non-admin users cannot toggle the whiteboard + if (!toggle) { + return null; + } + + return ( + + ); +}; +``` + +## Displaying the Collaborative Whiteboard + +You can display the whiteboard when it's open by embedding it as an iframe or as a React component for more fine-grained controls, if your app is built using React. + +```js +// Vanilla JavaScript Example +import { selectWhiteboard } from '@100mslive/hms-video-store'; + +const whiteboard = hmsStore.subscribe((whiteboard) => { + if (whiteboard?.open && whiteboard?.url) { + const whiteboardIframe = document.createElement('iframe'); + whiteboardIframe.src = whiteboard.url; + } else { + const whiteboardIframe = document.getElementById('whiteboard'); + whiteboardIframe?.remove(); + } +}, selectWhiteboard); +``` + +```jsx +// React Example +import React from 'react'; +import { useWhiteboard } from '@100mslive/react-sdk'; +import { Whiteboard } from '@100mslive/hms-whiteboard'; +import '@100mslive/hms-whiteboard/index.css'; + +const WhiteboardEmbed = () => { + const { token, endpoint } = useWhiteboard(); + + if (!token) { + return null; + } + + return ( +
+ { + console.log(store, editor); + }} + /> +
+ ); +}; +``` + +Whiteboard related CSS needs to be imported in your app's top level CSS files using `@import '@100mslive/hms-whiteboard/index.css';`(recommended) or in one of your top level JS file using `import '@100mslive/hms-whiteboard/index.css';`. + +Note that if you're using `@100mslive/roomkit-react` you'll need to import `@100mslive/roomkit-react/index.css` accordingly. diff --git a/packages/hms-whiteboard/package.json b/packages/hms-whiteboard/package.json new file mode 100644 index 0000000000..5692c97880 --- /dev/null +++ b/packages/hms-whiteboard/package.json @@ -0,0 +1,71 @@ +{ + "name": "@100mslive/hms-whiteboard", + "author": "100ms", + "license": "MIT", + "version": "0.0.15", + "main": "dist/index.cjs.js", + "module": "dist/index.js", + "types": "dist/index.d.ts", + "files": [ + "src", + "dist" + ], + "exports": { + ".": { + "require": "./dist/index.cjs.js", + "import": "./dist/index.js", + "default": "./dist/index.js" + }, + "./index.css": "./dist/index.css" + }, + "sideEffects": false, + "scripts": { + "prestart": "rm -rf dist", + "start": "rollup -c -w", + "build": "rm -rf dist && NODE_ENV=production rollup -c", + "build:proto": "protoc --ts_opt long_type_string --ts_out ./src/grpc -I=./proto proto/*proto", + "lint": "eslint -c .eslintrc src", + "lint:fix": "eslint -c .eslintrc src --fix", + "format": "prettier -w src/**" + }, + "dependencies": { + "@protobuf-ts/grpcweb-transport": "^2.9.1", + "@protobuf-ts/runtime": "^2.9.1", + "@protobuf-ts/runtime-rpc": "^2.9.1", + "@tldraw/tldraw": "2.0.0-alpha.19" + }, + "peerDependencies": { + "react": ">=17.0.2 <19.0.0", + "react-dom": ">=17.0.2 <19.0.0" + }, + "devDependencies": { + "@protobuf-ts/plugin": "^2.9.1", + "@rollup/plugin-commonjs": "^21.0.3", + "@rollup/plugin-node-resolve": "^13.1.3", + "@rollup/plugin-replace": "^5.0.1", + "@rollup/plugin-typescript": "^8.3.1", + "@types/node": "^20.12.5", + "@types/react": "^18.1.0", + "@types/react-dom": "^18.1.0", + "eslint": "^8.53.0", + "eslint-plugin-react-hooks": "^4.6.0", + "eslint-plugin-react-refresh": "^0.4.4", + "grpc-tools": "^1.12.4", + "react": "^18.1.0", + "react-dom": "^18.1.0", + "rollup": "^2.79.2", + "rollup-plugin-esbuild": "^4.9.3", + "rollup-plugin-import-css": "^3.5.0", + "rollup-plugin-terser": "^7.0.2", + "typescript": "^5.2.2" + }, + "keywords": [ + "whiteboard", + "tldraw", + "streaming", + "video", + "webrtc", + "conferencing", + "100ms" + ] +} diff --git a/packages/hms-whiteboard/proto/sessionstore.proto b/packages/hms-whiteboard/proto/sessionstore.proto new file mode 100644 index 0000000000..fc7190cbe8 --- /dev/null +++ b/packages/hms-whiteboard/proto/sessionstore.proto @@ -0,0 +1,129 @@ +syntax = "proto3"; + +package sessionstorepb; + +option go_package = "./internal;sessionstorepb"; + +import "google/protobuf/timestamp.proto"; + +service Api { + rpc Hello(HelloRequest) returns (HelloResponse) {} + rpc Subscribe(SubscribeRequest) returns (stream Event) {} +} + +message HelloRequest { + string name = 1; +} + +message HelloResponse { + string response = 1; +} + +message SubscribeRequest { + string name = 1; + int64 offset = 2; +} + +message Event { + string message = 1; + int64 sequence = 2; +} + +// metadata token -> session id, room id, user id, username + +// open is used for presence + +// change stream will return all keys in order of oldest to newsest. + +// max number of keys -> 5000 +// max size per key -> 10240 Bytes + +service Store { + // open - start listening to updates in keys with provided match patterns + // provide change_id as last received ID to resume updates + rpc open(OpenRequest) returns (stream ChangeStream) {} + + // get last stored value in given key + rpc get(GetRequest) returns (GetResponse) {} + + // set key value + rpc set(SetRequest) returns (SetResponse) {} + + // delete key from store + rpc delete (DeleteRequest) returns (DeleteResponse) {} + + // count get count of keys + rpc count(CountRequest) returns (CountResponse) {} +} + +// define new structure for value based on client needs and add support in +// following message +message Value { + enum Type { + NONE = 0; + BYTES = 1; + STRING = 2; + INTEGER = 3; + FLOAT = 4; + } + Type type = 1; + oneof Data { + int64 number = 2; + float float = 3; + string str = 4; + bytes raw_bytes = 5; + } +} + +message GetRequest { + string key = 1; +} + +message GetResponse { + string key = 1; + string namespace = 2; + Value value = 3; +} + +message DeleteRequest { + string key = 1; +} + +message DeleteResponse {} + +message SetRequest { + string key = 1; + Value value = 3; +} + +message SetResponse {} + +message ChangeStream { + string change_id = 1; + string key = 2; + string namespace = 3; + Value value = 4; + string from_id = 5; +} + +message Select { + oneof match { + string all = 1; // match all keys + string key = 2; // match key + string prefix = 3; // match keys with given prefix + string suffix = 4; // match keys with given suffix + } +} + +message OpenRequest { + // last received change_id for reconnection, "" if first connection + string change_id = 1; + repeated Select select = 3; +} + +message CountRequest {} + +message CountResponse { + int64 count = 1; +} + diff --git a/packages/roomkit-react/rollup.config.js b/packages/hms-whiteboard/rollup.config.js similarity index 72% rename from packages/roomkit-react/rollup.config.js rename to packages/hms-whiteboard/rollup.config.js index 4e1bb77026..80f752c7c1 100644 --- a/packages/roomkit-react/rollup.config.js +++ b/packages/hms-whiteboard/rollup.config.js @@ -1,5 +1,3 @@ -import image from '@rollup/plugin-image'; -import json from '@rollup/plugin-json'; import resolve from '@rollup/plugin-node-resolve'; import typescript from '@rollup/plugin-typescript'; import esbuild from 'rollup-plugin-esbuild'; @@ -17,18 +15,16 @@ const config = { input: 'src/index.ts', external: [...deps, ...peerDeps], output: [ - { file: pkg.main, format: 'cjs', sourcemap: true, inlineDynamicImports: true }, + { file: pkg.main, format: 'cjs', sourcemap: true }, { dir: 'dist', format: 'esm', preserveModules: true, preserveModulesRoot: 'src', sourcemap: true }, ], plugins: [ commonjs(), - css(), - image(), - json(), + css({ output: 'index.css' }), esbuild({ format: 'esm' }), - resolve({ preferBuiltins: false }), + resolve(), isProduction && terser(), - typescript({ sourceMap: true, exclude: '**/*.stories.tsx' }), + typescript({ sourceMap: true }), ], }; diff --git a/packages/hms-whiteboard/src/ErrorFallback.tsx b/packages/hms-whiteboard/src/ErrorFallback.tsx new file mode 100644 index 0000000000..2e62d0aec8 --- /dev/null +++ b/packages/hms-whiteboard/src/ErrorFallback.tsx @@ -0,0 +1,169 @@ +import React, { ComponentType, useEffect, useLayoutEffect, useRef, useState } from 'react'; +import { useValue } from '@tldraw/state'; +import { Editor, hardResetEditor } from '@tldraw/tldraw'; +import classNames from 'classnames'; + +const DASHBOARD_URL = 'https://dashboard.100ms.live/dashboard'; + +export type TLErrorFallbackComponent = ComponentType<{ + error: unknown; + refresh: () => void; + editor?: Editor; +}>; + +export const ErrorFallback: TLErrorFallbackComponent = ({ error, editor, refresh }) => { + const containerRef = useRef(null); + const [shouldShowError, setShouldShowError] = useState(process.env.NODE_ENV !== 'production'); + const [didCopy, setDidCopy] = useState(false); + const [shouldShowResetConfirmation, setShouldShowResetConfirmation] = useState(false); + + const errorMessage = error instanceof Error ? error.message : String(error); + const errorStack = error instanceof Error ? error.stack : null; + + const isDarkModeFromApp = useValue( + 'isDarkMode', + () => { + try { + if (editor) { + return editor.user.getIsDarkMode(); + } + } catch { + // we're in a funky error state so this might not work for spooky + // reasons. if not, we'll have another attempt later: + } + return null; + }, + [editor], + ); + const [ + , + // isDarkMode + setIsDarkMode, + ] = useState(null); + useLayoutEffect(() => { + // if we found a theme class from the app, we can just use that + if (isDarkModeFromApp !== null) { + setIsDarkMode(isDarkModeFromApp); + } + + // do any of our parents have a theme class? if yes then we can just + // rely on that and don't need to set our own class + let parent = containerRef.current?.parentElement; + let foundParentThemeClass = false; + while (parent) { + if (parent.classList.contains('tl-theme__dark') || parent.classList.contains('tl-theme__light')) { + foundParentThemeClass = true; + break; + } + parent = parent.parentElement; + } + if (foundParentThemeClass) { + setIsDarkMode(null); + return; + } + + // if we can't find a theme class from the app or from a parent, we have + // to fall back on using a media query: + setIsDarkMode(window.matchMedia('(prefers-color-scheme: dark)').matches); + }, [isDarkModeFromApp]); + + useEffect(() => { + if (didCopy) { + const timeout = setTimeout(() => { + setDidCopy(false); + }, 2000); + return () => clearTimeout(timeout); + } + }, [didCopy]); + + const copyError = () => { + const textarea = document.createElement('textarea'); + textarea.value = errorStack ?? errorMessage; + document.body.appendChild(textarea); + textarea.select(); + document.execCommand('copy'); + textarea.remove(); + setDidCopy(true); + }; + + const resetLocalState = async () => { + hardResetEditor(); + }; + + return ( +
+
+ {/* {editor && ( + // opportunistically attempt to render the canvas to reassure + // the user that their document is still there. there's a good + // chance this won't work (ie the error that we're currently + // notifying the user about originates in the canvas) so it's + // not a big deal if it doesn't work - in that case we just have + // a plain grey background. + null}> + +
+ +
+ + + )} */} +
+ {shouldShowResetConfirmation ? ( + <> +

Are you sure?

+

Resetting your data will delete your drawing and cannot be undone.

+
+ + +
+ + ) : ( + <> +

Something's gone wrong.

+

+ Sorry, we encountered an error. Please refresh the page to continue. If you keep seeing this error, you + can ask for help on Dashboard. +

+ {shouldShowError && ( +
+ 
+                  {errorStack ?? errorMessage}
+
+ +
+ )} +
+ +
+ +
+
+ + )} +
+
+ ); +}; diff --git a/packages/hms-whiteboard/src/Whiteboard.tsx b/packages/hms-whiteboard/src/Whiteboard.tsx new file mode 100644 index 0000000000..57b696cc3c --- /dev/null +++ b/packages/hms-whiteboard/src/Whiteboard.tsx @@ -0,0 +1,60 @@ +import React, { useState } from 'react'; +import { Editor, Tldraw } from '@tldraw/tldraw'; +import { ErrorFallback } from './ErrorFallback'; +import { useCollaboration } from './hooks/useCollaboration'; +import './index.css'; + +export interface WhiteboardProps { + endpoint?: string; + token: string; + zoomToContent?: boolean; + transparentCanvas?: boolean; + onMount?: (args: { store?: unknown; editor?: unknown }) => void; +} +export function Whiteboard(props: WhiteboardProps) { + const [key, setKey] = useState(Date.now() + props.token); + + return setKey(Date.now() + props.token)} {...props} />; +} + +function CollaborativeEditor({ + endpoint, + token, + zoomToContent, + transparentCanvas, + onMount, + refresh, +}: WhiteboardProps & { refresh: () => void }) { + const [editor, setEditor] = useState(); + const store = useCollaboration({ + endpoint, + token, + editor, + zoomToContent, + }); + + const handleMount = (editor: Editor) => { + setEditor(editor); + // @ts-expect-error - for debugging + window.editor = editor; + onMount?.({ store: store.store, editor }); + }; + + if (store.status === 'synced-remote' && store.connectionStatus === 'offline') { + return ; + } + + return ( + , + }} + hideUi={editor?.getInstanceState()?.isReadonly} + initialState={editor?.getInstanceState()?.isReadonly ? 'hand' : 'select'} + /> + ); +} diff --git a/packages/hms-whiteboard/src/grpc/sessionstore.client.ts b/packages/hms-whiteboard/src/grpc/sessionstore.client.ts new file mode 100644 index 0000000000..d5a584987b --- /dev/null +++ b/packages/hms-whiteboard/src/grpc/sessionstore.client.ts @@ -0,0 +1,174 @@ +// @generated by protobuf-ts 2.9.4 with parameter long_type_string +// @generated from protobuf file "sessionstore.proto" (package "sessionstorepb", syntax proto3) +// tslint:disable +import type { RpcOptions, RpcTransport, ServerStreamingCall, ServiceInfo, UnaryCall } from '@protobuf-ts/runtime-rpc'; +import { stackIntercept } from '@protobuf-ts/runtime-rpc'; +import type { + ChangeStream, + CountRequest, + CountResponse, + DeleteRequest, + DeleteResponse, + Event, + GetRequest, + GetResponse, + HelloRequest, + HelloResponse, + OpenRequest, + SetRequest, + SetResponse, + SubscribeRequest, +} from './sessionstore'; +import { Api, Store } from './sessionstore'; +/** + * @generated from protobuf service sessionstorepb.Api + */ +export interface IApiClient { + /** + * @generated from protobuf rpc: Hello(sessionstorepb.HelloRequest) returns (sessionstorepb.HelloResponse); + */ + hello(input: HelloRequest, options?: RpcOptions): UnaryCall; + /** + * @generated from protobuf rpc: Subscribe(sessionstorepb.SubscribeRequest) returns (stream sessionstorepb.Event); + */ + subscribe(input: SubscribeRequest, options?: RpcOptions): ServerStreamingCall; +} +/** + * @generated from protobuf service sessionstorepb.Api + */ +export class ApiClient implements IApiClient, ServiceInfo { + typeName = Api.typeName; + methods = Api.methods; + options = Api.options; + constructor(private readonly _transport: RpcTransport) {} + /** + * @generated from protobuf rpc: Hello(sessionstorepb.HelloRequest) returns (sessionstorepb.HelloResponse); + */ + hello(input: HelloRequest, options?: RpcOptions): UnaryCall { + const method = this.methods[0], + opt = this._transport.mergeOptions(options); + return stackIntercept('unary', this._transport, method, opt, input); + } + /** + * @generated from protobuf rpc: Subscribe(sessionstorepb.SubscribeRequest) returns (stream sessionstorepb.Event); + */ + subscribe(input: SubscribeRequest, options?: RpcOptions): ServerStreamingCall { + const method = this.methods[1], + opt = this._transport.mergeOptions(options); + return stackIntercept('serverStreaming', this._transport, method, opt, input); + } +} +// metadata token -> session id, room id, user id, username + +// open is used for presence + +// change stream will return all keys in order of oldest to newsest. + +// max number of keys -> 5000 +// max size per key -> 10240 Bytes + +/** + * @generated from protobuf service sessionstorepb.Store + */ +export interface IStoreClient { + /** + * open - start listening to updates in keys with provided match patterns + * provide change_id as last received ID to resume updates + * + * @generated from protobuf rpc: open(sessionstorepb.OpenRequest) returns (stream sessionstorepb.ChangeStream); + */ + open(input: OpenRequest, options?: RpcOptions): ServerStreamingCall; + /** + * get last stored value in given key + * + * @generated from protobuf rpc: get(sessionstorepb.GetRequest) returns (sessionstorepb.GetResponse); + */ + get(input: GetRequest, options?: RpcOptions): UnaryCall; + /** + * set key value + * + * @generated from protobuf rpc: set(sessionstorepb.SetRequest) returns (sessionstorepb.SetResponse); + */ + set(input: SetRequest, options?: RpcOptions): UnaryCall; + /** + * delete key from store + * + * @generated from protobuf rpc: delete(sessionstorepb.DeleteRequest) returns (sessionstorepb.DeleteResponse); + */ + delete(input: DeleteRequest, options?: RpcOptions): UnaryCall; + /** + * count get count of keys + * + * @generated from protobuf rpc: count(sessionstorepb.CountRequest) returns (sessionstorepb.CountResponse); + */ + count(input: CountRequest, options?: RpcOptions): UnaryCall; +} +// metadata token -> session id, room id, user id, username + +// open is used for presence + +// change stream will return all keys in order of oldest to newsest. + +// max number of keys -> 5000 +// max size per key -> 10240 Bytes + +/** + * @generated from protobuf service sessionstorepb.Store + */ +export class StoreClient implements IStoreClient, ServiceInfo { + typeName = Store.typeName; + methods = Store.methods; + options = Store.options; + constructor(private readonly _transport: RpcTransport) {} + /** + * open - start listening to updates in keys with provided match patterns + * provide change_id as last received ID to resume updates + * + * @generated from protobuf rpc: open(sessionstorepb.OpenRequest) returns (stream sessionstorepb.ChangeStream); + */ + open(input: OpenRequest, options?: RpcOptions): ServerStreamingCall { + const method = this.methods[0], + opt = this._transport.mergeOptions(options); + return stackIntercept('serverStreaming', this._transport, method, opt, input); + } + /** + * get last stored value in given key + * + * @generated from protobuf rpc: get(sessionstorepb.GetRequest) returns (sessionstorepb.GetResponse); + */ + get(input: GetRequest, options?: RpcOptions): UnaryCall { + const method = this.methods[1], + opt = this._transport.mergeOptions(options); + return stackIntercept('unary', this._transport, method, opt, input); + } + /** + * set key value + * + * @generated from protobuf rpc: set(sessionstorepb.SetRequest) returns (sessionstorepb.SetResponse); + */ + set(input: SetRequest, options?: RpcOptions): UnaryCall { + const method = this.methods[2], + opt = this._transport.mergeOptions(options); + return stackIntercept('unary', this._transport, method, opt, input); + } + /** + * delete key from store + * + * @generated from protobuf rpc: delete(sessionstorepb.DeleteRequest) returns (sessionstorepb.DeleteResponse); + */ + delete(input: DeleteRequest, options?: RpcOptions): UnaryCall { + const method = this.methods[3], + opt = this._transport.mergeOptions(options); + return stackIntercept('unary', this._transport, method, opt, input); + } + /** + * count get count of keys + * + * @generated from protobuf rpc: count(sessionstorepb.CountRequest) returns (sessionstorepb.CountResponse); + */ + count(input: CountRequest, options?: RpcOptions): UnaryCall { + const method = this.methods[4], + opt = this._transport.mergeOptions(options); + return stackIntercept('unary', this._transport, method, opt, input); + } +} diff --git a/packages/hms-whiteboard/src/grpc/sessionstore.ts b/packages/hms-whiteboard/src/grpc/sessionstore.ts new file mode 100644 index 0000000000..62c29f3d98 --- /dev/null +++ b/packages/hms-whiteboard/src/grpc/sessionstore.ts @@ -0,0 +1,1128 @@ +// @generated by protobuf-ts 2.9.4 with parameter long_type_string +// @generated from protobuf file "sessionstore.proto" (package "sessionstorepb", syntax proto3) +// tslint:disable +import type { + BinaryReadOptions, + BinaryWriteOptions, + IBinaryReader, + IBinaryWriter, + PartialMessage, +} from '@protobuf-ts/runtime'; +import { MessageType, UnknownFieldHandler, WireType, reflectionMergePartial } from '@protobuf-ts/runtime'; +import { ServiceType } from '@protobuf-ts/runtime-rpc'; +/** + * @generated from protobuf message sessionstorepb.HelloRequest + */ +export interface HelloRequest { + /** + * @generated from protobuf field: string name = 1; + */ + name: string; +} +/** + * @generated from protobuf message sessionstorepb.HelloResponse + */ +export interface HelloResponse { + /** + * @generated from protobuf field: string response = 1; + */ + response: string; +} +/** + * @generated from protobuf message sessionstorepb.SubscribeRequest + */ +export interface SubscribeRequest { + /** + * @generated from protobuf field: string name = 1; + */ + name: string; + /** + * @generated from protobuf field: int64 offset = 2; + */ + offset: string; +} +/** + * @generated from protobuf message sessionstorepb.Event + */ +export interface Event { + /** + * @generated from protobuf field: string message = 1; + */ + message: string; + /** + * @generated from protobuf field: int64 sequence = 2; + */ + sequence: string; +} +/** + * define new structure for value based on client needs and add support in + * following message + * + * @generated from protobuf message sessionstorepb.Value + */ +export interface Value { + /** + * @generated from protobuf field: sessionstorepb.Value.Type type = 1; + */ + type: Value_Type; + /** + * @generated from protobuf oneof: Data + */ + data: + | { + oneofKind: 'number'; + /** + * @generated from protobuf field: int64 number = 2; + */ + number: string; + } + | { + oneofKind: 'float'; + /** + * @generated from protobuf field: float float = 3; + */ + float: number; + } + | { + oneofKind: 'str'; + /** + * @generated from protobuf field: string str = 4; + */ + str: string; + } + | { + oneofKind: 'rawBytes'; + /** + * @generated from protobuf field: bytes raw_bytes = 5; + */ + rawBytes: Uint8Array; + } + | { + oneofKind: undefined; + }; +} +/** + * @generated from protobuf enum sessionstorepb.Value.Type + */ +export enum Value_Type { + /** + * @generated from protobuf enum value: NONE = 0; + */ + NONE = 0, + /** + * @generated from protobuf enum value: BYTES = 1; + */ + BYTES = 1, + /** + * @generated from protobuf enum value: STRING = 2; + */ + STRING = 2, + /** + * @generated from protobuf enum value: INTEGER = 3; + */ + INTEGER = 3, + /** + * @generated from protobuf enum value: FLOAT = 4; + */ + FLOAT = 4, +} +/** + * @generated from protobuf message sessionstorepb.GetRequest + */ +export interface GetRequest { + /** + * @generated from protobuf field: string key = 1; + */ + key: string; +} +/** + * @generated from protobuf message sessionstorepb.GetResponse + */ +export interface GetResponse { + /** + * @generated from protobuf field: string key = 1; + */ + key: string; + /** + * @generated from protobuf field: string namespace = 2; + */ + namespace: string; + /** + * @generated from protobuf field: sessionstorepb.Value value = 3; + */ + value?: Value; +} +/** + * @generated from protobuf message sessionstorepb.DeleteRequest + */ +export interface DeleteRequest { + /** + * @generated from protobuf field: string key = 1; + */ + key: string; +} +/** + * @generated from protobuf message sessionstorepb.DeleteResponse + */ +export interface DeleteResponse {} +/** + * @generated from protobuf message sessionstorepb.SetRequest + */ +export interface SetRequest { + /** + * @generated from protobuf field: string key = 1; + */ + key: string; + /** + * @generated from protobuf field: sessionstorepb.Value value = 3; + */ + value?: Value; +} +/** + * @generated from protobuf message sessionstorepb.SetResponse + */ +export interface SetResponse {} +/** + * @generated from protobuf message sessionstorepb.ChangeStream + */ +export interface ChangeStream { + /** + * @generated from protobuf field: string change_id = 1; + */ + changeId: string; + /** + * @generated from protobuf field: string key = 2; + */ + key: string; + /** + * @generated from protobuf field: string namespace = 3; + */ + namespace: string; + /** + * @generated from protobuf field: sessionstorepb.Value value = 4; + */ + value?: Value; + /** + * @generated from protobuf field: string from_id = 5; + */ + fromId: string; +} +/** + * @generated from protobuf message sessionstorepb.Select + */ +export interface Select { + /** + * @generated from protobuf oneof: match + */ + match: + | { + oneofKind: 'all'; + /** + * @generated from protobuf field: string all = 1; + */ + all: string; // match all keys + } + | { + oneofKind: 'key'; + /** + * @generated from protobuf field: string key = 2; + */ + key: string; // match key + } + | { + oneofKind: 'prefix'; + /** + * @generated from protobuf field: string prefix = 3; + */ + prefix: string; // match keys with given prefix + } + | { + oneofKind: 'suffix'; + /** + * @generated from protobuf field: string suffix = 4; + */ + suffix: string; // match keys with given suffix + } + | { + oneofKind: undefined; + }; +} +/** + * @generated from protobuf message sessionstorepb.OpenRequest + */ +export interface OpenRequest { + /** + * last received change_id for reconnection, "" if first connection + * + * @generated from protobuf field: string change_id = 1; + */ + changeId: string; + /** + * @generated from protobuf field: repeated sessionstorepb.Select select = 3; + */ + select: Select[]; +} +/** + * @generated from protobuf message sessionstorepb.CountRequest + */ +export interface CountRequest {} +/** + * @generated from protobuf message sessionstorepb.CountResponse + */ +export interface CountResponse { + /** + * @generated from protobuf field: int64 count = 1; + */ + count: string; +} +// @generated message type with reflection information, may provide speed optimized methods +class HelloRequest$Type extends MessageType { + constructor() { + super('sessionstorepb.HelloRequest', [{ no: 1, name: 'name', kind: 'scalar', T: 9 /*ScalarType.STRING*/ }]); + } + create(value?: PartialMessage): HelloRequest { + const message = globalThis.Object.create(this.messagePrototype!); + message.name = ''; + if (value !== undefined) reflectionMergePartial(this, message, value); + return message; + } + internalBinaryRead( + reader: IBinaryReader, + length: number, + options: BinaryReadOptions, + target?: HelloRequest, + ): HelloRequest { + let message = target ?? this.create(), + end = reader.pos + length; + while (reader.pos < end) { + let [fieldNo, wireType] = reader.tag(); + switch (fieldNo) { + case /* string name */ 1: + message.name = reader.string(); + break; + default: + let u = options.readUnknownField; + if (u === 'throw') + throw new globalThis.Error(`Unknown field ${fieldNo} (wire type ${wireType}) for ${this.typeName}`); + let d = reader.skip(wireType); + if (u !== false) (u === true ? UnknownFieldHandler.onRead : u)(this.typeName, message, fieldNo, wireType, d); + } + } + return message; + } + internalBinaryWrite(message: HelloRequest, writer: IBinaryWriter, options: BinaryWriteOptions): IBinaryWriter { + /* string name = 1; */ + if (message.name !== '') writer.tag(1, WireType.LengthDelimited).string(message.name); + let u = options.writeUnknownFields; + if (u !== false) (u == true ? UnknownFieldHandler.onWrite : u)(this.typeName, message, writer); + return writer; + } +} +/** + * @generated MessageType for protobuf message sessionstorepb.HelloRequest + */ +export const HelloRequest = new HelloRequest$Type(); +// @generated message type with reflection information, may provide speed optimized methods +class HelloResponse$Type extends MessageType { + constructor() { + super('sessionstorepb.HelloResponse', [{ no: 1, name: 'response', kind: 'scalar', T: 9 /*ScalarType.STRING*/ }]); + } + create(value?: PartialMessage): HelloResponse { + const message = globalThis.Object.create(this.messagePrototype!); + message.response = ''; + if (value !== undefined) reflectionMergePartial(this, message, value); + return message; + } + internalBinaryRead( + reader: IBinaryReader, + length: number, + options: BinaryReadOptions, + target?: HelloResponse, + ): HelloResponse { + let message = target ?? this.create(), + end = reader.pos + length; + while (reader.pos < end) { + let [fieldNo, wireType] = reader.tag(); + switch (fieldNo) { + case /* string response */ 1: + message.response = reader.string(); + break; + default: + let u = options.readUnknownField; + if (u === 'throw') + throw new globalThis.Error(`Unknown field ${fieldNo} (wire type ${wireType}) for ${this.typeName}`); + let d = reader.skip(wireType); + if (u !== false) (u === true ? UnknownFieldHandler.onRead : u)(this.typeName, message, fieldNo, wireType, d); + } + } + return message; + } + internalBinaryWrite(message: HelloResponse, writer: IBinaryWriter, options: BinaryWriteOptions): IBinaryWriter { + /* string response = 1; */ + if (message.response !== '') writer.tag(1, WireType.LengthDelimited).string(message.response); + let u = options.writeUnknownFields; + if (u !== false) (u == true ? UnknownFieldHandler.onWrite : u)(this.typeName, message, writer); + return writer; + } +} +/** + * @generated MessageType for protobuf message sessionstorepb.HelloResponse + */ +export const HelloResponse = new HelloResponse$Type(); +// @generated message type with reflection information, may provide speed optimized methods +class SubscribeRequest$Type extends MessageType { + constructor() { + super('sessionstorepb.SubscribeRequest', [ + { no: 1, name: 'name', kind: 'scalar', T: 9 /*ScalarType.STRING*/ }, + { no: 2, name: 'offset', kind: 'scalar', T: 3 /*ScalarType.INT64*/ }, + ]); + } + create(value?: PartialMessage): SubscribeRequest { + const message = globalThis.Object.create(this.messagePrototype!); + message.name = ''; + message.offset = '0'; + if (value !== undefined) reflectionMergePartial(this, message, value); + return message; + } + internalBinaryRead( + reader: IBinaryReader, + length: number, + options: BinaryReadOptions, + target?: SubscribeRequest, + ): SubscribeRequest { + let message = target ?? this.create(), + end = reader.pos + length; + while (reader.pos < end) { + let [fieldNo, wireType] = reader.tag(); + switch (fieldNo) { + case /* string name */ 1: + message.name = reader.string(); + break; + case /* int64 offset */ 2: + message.offset = reader.int64().toString(); + break; + default: + let u = options.readUnknownField; + if (u === 'throw') + throw new globalThis.Error(`Unknown field ${fieldNo} (wire type ${wireType}) for ${this.typeName}`); + let d = reader.skip(wireType); + if (u !== false) (u === true ? UnknownFieldHandler.onRead : u)(this.typeName, message, fieldNo, wireType, d); + } + } + return message; + } + internalBinaryWrite(message: SubscribeRequest, writer: IBinaryWriter, options: BinaryWriteOptions): IBinaryWriter { + /* string name = 1; */ + if (message.name !== '') writer.tag(1, WireType.LengthDelimited).string(message.name); + /* int64 offset = 2; */ + if (message.offset !== '0') writer.tag(2, WireType.Varint).int64(message.offset); + let u = options.writeUnknownFields; + if (u !== false) (u == true ? UnknownFieldHandler.onWrite : u)(this.typeName, message, writer); + return writer; + } +} +/** + * @generated MessageType for protobuf message sessionstorepb.SubscribeRequest + */ +export const SubscribeRequest = new SubscribeRequest$Type(); +// @generated message type with reflection information, may provide speed optimized methods +class Event$Type extends MessageType { + constructor() { + super('sessionstorepb.Event', [ + { no: 1, name: 'message', kind: 'scalar', T: 9 /*ScalarType.STRING*/ }, + { no: 2, name: 'sequence', kind: 'scalar', T: 3 /*ScalarType.INT64*/ }, + ]); + } + create(value?: PartialMessage): Event { + const message = globalThis.Object.create(this.messagePrototype!); + message.message = ''; + message.sequence = '0'; + if (value !== undefined) reflectionMergePartial(this, message, value); + return message; + } + internalBinaryRead(reader: IBinaryReader, length: number, options: BinaryReadOptions, target?: Event): Event { + let message = target ?? this.create(), + end = reader.pos + length; + while (reader.pos < end) { + let [fieldNo, wireType] = reader.tag(); + switch (fieldNo) { + case /* string message */ 1: + message.message = reader.string(); + break; + case /* int64 sequence */ 2: + message.sequence = reader.int64().toString(); + break; + default: + let u = options.readUnknownField; + if (u === 'throw') + throw new globalThis.Error(`Unknown field ${fieldNo} (wire type ${wireType}) for ${this.typeName}`); + let d = reader.skip(wireType); + if (u !== false) (u === true ? UnknownFieldHandler.onRead : u)(this.typeName, message, fieldNo, wireType, d); + } + } + return message; + } + internalBinaryWrite(message: Event, writer: IBinaryWriter, options: BinaryWriteOptions): IBinaryWriter { + /* string message = 1; */ + if (message.message !== '') writer.tag(1, WireType.LengthDelimited).string(message.message); + /* int64 sequence = 2; */ + if (message.sequence !== '0') writer.tag(2, WireType.Varint).int64(message.sequence); + let u = options.writeUnknownFields; + if (u !== false) (u == true ? UnknownFieldHandler.onWrite : u)(this.typeName, message, writer); + return writer; + } +} +/** + * @generated MessageType for protobuf message sessionstorepb.Event + */ +export const Event = new Event$Type(); +// @generated message type with reflection information, may provide speed optimized methods +class Value$Type extends MessageType { + constructor() { + super('sessionstorepb.Value', [ + { no: 1, name: 'type', kind: 'enum', T: () => ['sessionstorepb.Value.Type', Value_Type] }, + { no: 2, name: 'number', kind: 'scalar', oneof: 'data', T: 3 /*ScalarType.INT64*/ }, + { no: 3, name: 'float', kind: 'scalar', oneof: 'data', T: 2 /*ScalarType.FLOAT*/ }, + { no: 4, name: 'str', kind: 'scalar', oneof: 'data', T: 9 /*ScalarType.STRING*/ }, + { no: 5, name: 'raw_bytes', kind: 'scalar', oneof: 'data', T: 12 /*ScalarType.BYTES*/ }, + ]); + } + create(value?: PartialMessage): Value { + const message = globalThis.Object.create(this.messagePrototype!); + message.type = 0; + message.data = { oneofKind: undefined }; + if (value !== undefined) reflectionMergePartial(this, message, value); + return message; + } + internalBinaryRead(reader: IBinaryReader, length: number, options: BinaryReadOptions, target?: Value): Value { + let message = target ?? this.create(), + end = reader.pos + length; + while (reader.pos < end) { + let [fieldNo, wireType] = reader.tag(); + switch (fieldNo) { + case /* sessionstorepb.Value.Type type */ 1: + message.type = reader.int32(); + break; + case /* int64 number */ 2: + message.data = { + oneofKind: 'number', + number: reader.int64().toString(), + }; + break; + case /* float float */ 3: + message.data = { + oneofKind: 'float', + float: reader.float(), + }; + break; + case /* string str */ 4: + message.data = { + oneofKind: 'str', + str: reader.string(), + }; + break; + case /* bytes raw_bytes */ 5: + message.data = { + oneofKind: 'rawBytes', + rawBytes: reader.bytes(), + }; + break; + default: + let u = options.readUnknownField; + if (u === 'throw') + throw new globalThis.Error(`Unknown field ${fieldNo} (wire type ${wireType}) for ${this.typeName}`); + let d = reader.skip(wireType); + if (u !== false) (u === true ? UnknownFieldHandler.onRead : u)(this.typeName, message, fieldNo, wireType, d); + } + } + return message; + } + internalBinaryWrite(message: Value, writer: IBinaryWriter, options: BinaryWriteOptions): IBinaryWriter { + /* sessionstorepb.Value.Type type = 1; */ + if (message.type !== 0) writer.tag(1, WireType.Varint).int32(message.type); + /* int64 number = 2; */ + if (message.data.oneofKind === 'number') writer.tag(2, WireType.Varint).int64(message.data.number); + /* float float = 3; */ + if (message.data.oneofKind === 'float') writer.tag(3, WireType.Bit32).float(message.data.float); + /* string str = 4; */ + if (message.data.oneofKind === 'str') writer.tag(4, WireType.LengthDelimited).string(message.data.str); + /* bytes raw_bytes = 5; */ + if (message.data.oneofKind === 'rawBytes') writer.tag(5, WireType.LengthDelimited).bytes(message.data.rawBytes); + let u = options.writeUnknownFields; + if (u !== false) (u == true ? UnknownFieldHandler.onWrite : u)(this.typeName, message, writer); + return writer; + } +} +/** + * @generated MessageType for protobuf message sessionstorepb.Value + */ +export const Value = new Value$Type(); +// @generated message type with reflection information, may provide speed optimized methods +class GetRequest$Type extends MessageType { + constructor() { + super('sessionstorepb.GetRequest', [{ no: 1, name: 'key', kind: 'scalar', T: 9 /*ScalarType.STRING*/ }]); + } + create(value?: PartialMessage): GetRequest { + const message = globalThis.Object.create(this.messagePrototype!); + message.key = ''; + if (value !== undefined) reflectionMergePartial(this, message, value); + return message; + } + internalBinaryRead( + reader: IBinaryReader, + length: number, + options: BinaryReadOptions, + target?: GetRequest, + ): GetRequest { + let message = target ?? this.create(), + end = reader.pos + length; + while (reader.pos < end) { + let [fieldNo, wireType] = reader.tag(); + switch (fieldNo) { + case /* string key */ 1: + message.key = reader.string(); + break; + default: + let u = options.readUnknownField; + if (u === 'throw') + throw new globalThis.Error(`Unknown field ${fieldNo} (wire type ${wireType}) for ${this.typeName}`); + let d = reader.skip(wireType); + if (u !== false) (u === true ? UnknownFieldHandler.onRead : u)(this.typeName, message, fieldNo, wireType, d); + } + } + return message; + } + internalBinaryWrite(message: GetRequest, writer: IBinaryWriter, options: BinaryWriteOptions): IBinaryWriter { + /* string key = 1; */ + if (message.key !== '') writer.tag(1, WireType.LengthDelimited).string(message.key); + let u = options.writeUnknownFields; + if (u !== false) (u == true ? UnknownFieldHandler.onWrite : u)(this.typeName, message, writer); + return writer; + } +} +/** + * @generated MessageType for protobuf message sessionstorepb.GetRequest + */ +export const GetRequest = new GetRequest$Type(); +// @generated message type with reflection information, may provide speed optimized methods +class GetResponse$Type extends MessageType { + constructor() { + super('sessionstorepb.GetResponse', [ + { no: 1, name: 'key', kind: 'scalar', T: 9 /*ScalarType.STRING*/ }, + { no: 2, name: 'namespace', kind: 'scalar', T: 9 /*ScalarType.STRING*/ }, + { no: 3, name: 'value', kind: 'message', T: () => Value }, + ]); + } + create(value?: PartialMessage): GetResponse { + const message = globalThis.Object.create(this.messagePrototype!); + message.key = ''; + message.namespace = ''; + if (value !== undefined) reflectionMergePartial(this, message, value); + return message; + } + internalBinaryRead( + reader: IBinaryReader, + length: number, + options: BinaryReadOptions, + target?: GetResponse, + ): GetResponse { + let message = target ?? this.create(), + end = reader.pos + length; + while (reader.pos < end) { + let [fieldNo, wireType] = reader.tag(); + switch (fieldNo) { + case /* string key */ 1: + message.key = reader.string(); + break; + case /* string namespace */ 2: + message.namespace = reader.string(); + break; + case /* sessionstorepb.Value value */ 3: + message.value = Value.internalBinaryRead(reader, reader.uint32(), options, message.value); + break; + default: + let u = options.readUnknownField; + if (u === 'throw') + throw new globalThis.Error(`Unknown field ${fieldNo} (wire type ${wireType}) for ${this.typeName}`); + let d = reader.skip(wireType); + if (u !== false) (u === true ? UnknownFieldHandler.onRead : u)(this.typeName, message, fieldNo, wireType, d); + } + } + return message; + } + internalBinaryWrite(message: GetResponse, writer: IBinaryWriter, options: BinaryWriteOptions): IBinaryWriter { + /* string key = 1; */ + if (message.key !== '') writer.tag(1, WireType.LengthDelimited).string(message.key); + /* string namespace = 2; */ + if (message.namespace !== '') writer.tag(2, WireType.LengthDelimited).string(message.namespace); + /* sessionstorepb.Value value = 3; */ + if (message.value) + Value.internalBinaryWrite(message.value, writer.tag(3, WireType.LengthDelimited).fork(), options).join(); + let u = options.writeUnknownFields; + if (u !== false) (u == true ? UnknownFieldHandler.onWrite : u)(this.typeName, message, writer); + return writer; + } +} +/** + * @generated MessageType for protobuf message sessionstorepb.GetResponse + */ +export const GetResponse = new GetResponse$Type(); +// @generated message type with reflection information, may provide speed optimized methods +class DeleteRequest$Type extends MessageType { + constructor() { + super('sessionstorepb.DeleteRequest', [{ no: 1, name: 'key', kind: 'scalar', T: 9 /*ScalarType.STRING*/ }]); + } + create(value?: PartialMessage): DeleteRequest { + const message = globalThis.Object.create(this.messagePrototype!); + message.key = ''; + if (value !== undefined) reflectionMergePartial(this, message, value); + return message; + } + internalBinaryRead( + reader: IBinaryReader, + length: number, + options: BinaryReadOptions, + target?: DeleteRequest, + ): DeleteRequest { + let message = target ?? this.create(), + end = reader.pos + length; + while (reader.pos < end) { + let [fieldNo, wireType] = reader.tag(); + switch (fieldNo) { + case /* string key */ 1: + message.key = reader.string(); + break; + default: + let u = options.readUnknownField; + if (u === 'throw') + throw new globalThis.Error(`Unknown field ${fieldNo} (wire type ${wireType}) for ${this.typeName}`); + let d = reader.skip(wireType); + if (u !== false) (u === true ? UnknownFieldHandler.onRead : u)(this.typeName, message, fieldNo, wireType, d); + } + } + return message; + } + internalBinaryWrite(message: DeleteRequest, writer: IBinaryWriter, options: BinaryWriteOptions): IBinaryWriter { + /* string key = 1; */ + if (message.key !== '') writer.tag(1, WireType.LengthDelimited).string(message.key); + let u = options.writeUnknownFields; + if (u !== false) (u == true ? UnknownFieldHandler.onWrite : u)(this.typeName, message, writer); + return writer; + } +} +/** + * @generated MessageType for protobuf message sessionstorepb.DeleteRequest + */ +export const DeleteRequest = new DeleteRequest$Type(); +// @generated message type with reflection information, may provide speed optimized methods +class DeleteResponse$Type extends MessageType { + constructor() { + super('sessionstorepb.DeleteResponse', []); + } + create(value?: PartialMessage): DeleteResponse { + const message = globalThis.Object.create(this.messagePrototype!); + if (value !== undefined) reflectionMergePartial(this, message, value); + return message; + } + internalBinaryRead( + reader: IBinaryReader, + length: number, + options: BinaryReadOptions, + target?: DeleteResponse, + ): DeleteResponse { + return target ?? this.create(); + } + internalBinaryWrite(message: DeleteResponse, writer: IBinaryWriter, options: BinaryWriteOptions): IBinaryWriter { + let u = options.writeUnknownFields; + if (u !== false) (u == true ? UnknownFieldHandler.onWrite : u)(this.typeName, message, writer); + return writer; + } +} +/** + * @generated MessageType for protobuf message sessionstorepb.DeleteResponse + */ +export const DeleteResponse = new DeleteResponse$Type(); +// @generated message type with reflection information, may provide speed optimized methods +class SetRequest$Type extends MessageType { + constructor() { + super('sessionstorepb.SetRequest', [ + { no: 1, name: 'key', kind: 'scalar', T: 9 /*ScalarType.STRING*/ }, + { no: 3, name: 'value', kind: 'message', T: () => Value }, + ]); + } + create(value?: PartialMessage): SetRequest { + const message = globalThis.Object.create(this.messagePrototype!); + message.key = ''; + if (value !== undefined) reflectionMergePartial(this, message, value); + return message; + } + internalBinaryRead( + reader: IBinaryReader, + length: number, + options: BinaryReadOptions, + target?: SetRequest, + ): SetRequest { + let message = target ?? this.create(), + end = reader.pos + length; + while (reader.pos < end) { + let [fieldNo, wireType] = reader.tag(); + switch (fieldNo) { + case /* string key */ 1: + message.key = reader.string(); + break; + case /* sessionstorepb.Value value */ 3: + message.value = Value.internalBinaryRead(reader, reader.uint32(), options, message.value); + break; + default: + let u = options.readUnknownField; + if (u === 'throw') + throw new globalThis.Error(`Unknown field ${fieldNo} (wire type ${wireType}) for ${this.typeName}`); + let d = reader.skip(wireType); + if (u !== false) (u === true ? UnknownFieldHandler.onRead : u)(this.typeName, message, fieldNo, wireType, d); + } + } + return message; + } + internalBinaryWrite(message: SetRequest, writer: IBinaryWriter, options: BinaryWriteOptions): IBinaryWriter { + /* string key = 1; */ + if (message.key !== '') writer.tag(1, WireType.LengthDelimited).string(message.key); + /* sessionstorepb.Value value = 3; */ + if (message.value) + Value.internalBinaryWrite(message.value, writer.tag(3, WireType.LengthDelimited).fork(), options).join(); + let u = options.writeUnknownFields; + if (u !== false) (u == true ? UnknownFieldHandler.onWrite : u)(this.typeName, message, writer); + return writer; + } +} +/** + * @generated MessageType for protobuf message sessionstorepb.SetRequest + */ +export const SetRequest = new SetRequest$Type(); +// @generated message type with reflection information, may provide speed optimized methods +class SetResponse$Type extends MessageType { + constructor() { + super('sessionstorepb.SetResponse', []); + } + create(value?: PartialMessage): SetResponse { + const message = globalThis.Object.create(this.messagePrototype!); + if (value !== undefined) reflectionMergePartial(this, message, value); + return message; + } + internalBinaryRead( + reader: IBinaryReader, + length: number, + options: BinaryReadOptions, + target?: SetResponse, + ): SetResponse { + return target ?? this.create(); + } + internalBinaryWrite(message: SetResponse, writer: IBinaryWriter, options: BinaryWriteOptions): IBinaryWriter { + let u = options.writeUnknownFields; + if (u !== false) (u == true ? UnknownFieldHandler.onWrite : u)(this.typeName, message, writer); + return writer; + } +} +/** + * @generated MessageType for protobuf message sessionstorepb.SetResponse + */ +export const SetResponse = new SetResponse$Type(); +// @generated message type with reflection information, may provide speed optimized methods +class ChangeStream$Type extends MessageType { + constructor() { + super('sessionstorepb.ChangeStream', [ + { no: 1, name: 'change_id', kind: 'scalar', T: 9 /*ScalarType.STRING*/ }, + { no: 2, name: 'key', kind: 'scalar', T: 9 /*ScalarType.STRING*/ }, + { no: 3, name: 'namespace', kind: 'scalar', T: 9 /*ScalarType.STRING*/ }, + { no: 4, name: 'value', kind: 'message', T: () => Value }, + { no: 5, name: 'from_id', kind: 'scalar', T: 9 /*ScalarType.STRING*/ }, + ]); + } + create(value?: PartialMessage): ChangeStream { + const message = globalThis.Object.create(this.messagePrototype!); + message.changeId = ''; + message.key = ''; + message.namespace = ''; + message.fromId = ''; + if (value !== undefined) reflectionMergePartial(this, message, value); + return message; + } + internalBinaryRead( + reader: IBinaryReader, + length: number, + options: BinaryReadOptions, + target?: ChangeStream, + ): ChangeStream { + let message = target ?? this.create(), + end = reader.pos + length; + while (reader.pos < end) { + let [fieldNo, wireType] = reader.tag(); + switch (fieldNo) { + case /* string change_id */ 1: + message.changeId = reader.string(); + break; + case /* string key */ 2: + message.key = reader.string(); + break; + case /* string namespace */ 3: + message.namespace = reader.string(); + break; + case /* sessionstorepb.Value value */ 4: + message.value = Value.internalBinaryRead(reader, reader.uint32(), options, message.value); + break; + case /* string from_id */ 5: + message.fromId = reader.string(); + break; + default: + let u = options.readUnknownField; + if (u === 'throw') + throw new globalThis.Error(`Unknown field ${fieldNo} (wire type ${wireType}) for ${this.typeName}`); + let d = reader.skip(wireType); + if (u !== false) (u === true ? UnknownFieldHandler.onRead : u)(this.typeName, message, fieldNo, wireType, d); + } + } + return message; + } + internalBinaryWrite(message: ChangeStream, writer: IBinaryWriter, options: BinaryWriteOptions): IBinaryWriter { + /* string change_id = 1; */ + if (message.changeId !== '') writer.tag(1, WireType.LengthDelimited).string(message.changeId); + /* string key = 2; */ + if (message.key !== '') writer.tag(2, WireType.LengthDelimited).string(message.key); + /* string namespace = 3; */ + if (message.namespace !== '') writer.tag(3, WireType.LengthDelimited).string(message.namespace); + /* sessionstorepb.Value value = 4; */ + if (message.value) + Value.internalBinaryWrite(message.value, writer.tag(4, WireType.LengthDelimited).fork(), options).join(); + /* string from_id = 5; */ + if (message.fromId !== '') writer.tag(5, WireType.LengthDelimited).string(message.fromId); + let u = options.writeUnknownFields; + if (u !== false) (u == true ? UnknownFieldHandler.onWrite : u)(this.typeName, message, writer); + return writer; + } +} +/** + * @generated MessageType for protobuf message sessionstorepb.ChangeStream + */ +export const ChangeStream = new ChangeStream$Type(); +// @generated message type with reflection information, may provide speed optimized methods +class Select$Type extends MessageType): Select { + const message = globalThis.Object.create(this.messagePrototype!); + message.match = { oneofKind: undefined }; + if (value !== undefined) reflectionMergePartial